feat(map): add cache-first Cesium gateway and AMD egress

2026-07-16 02:23:56 +03:00
parent 59e9c92415
commit 7b55d887bc
31 changed files with 4956 additions and 167 deletions
@@ -0,0 +1,177 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
+import { createServer } from "node:http";
+import { createServer as createNetServer } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { gzipSync } from "node:zlib";
+
+const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-live-cache-smoke-"));
+const upstreamPort = await freePort();
+const gatewayPort = await freePort();
+let upstream;
+let gateway;
+let upstreamState = "live-1";
+let gzipRevision = 1;
+let upstreamRequests = 0;
+
+try {
+  upstream = createServer((request, response) => {
+    upstreamRequests += 1;
+    if (request.url?.startsWith("/gzip/")) {
+      const compressed = gzipSync(JSON.stringify({ revision: gzipRevision, payload: "x".repeat(8_192) }));
+      response.writeHead(200, {
+        "content-type": "application/json",
+        "content-encoding": "gzip",
+        "content-length": String(compressed.byteLength),
+        "cache-control": "max-age=3600",
+      });
+      response.end(compressed);
+      return;
+    }
+    if (upstreamState === "unavailable") {
+      response.writeHead(503, { "content-type": "text/plain" });
+      response.end("unavailable");
+      return;
+    }
+    response.writeHead(200, { "content-type": "text/plain", "cache-control": "max-age=3600" });
+    response.end(upstreamState);
+  });
+  upstream.listen(upstreamPort, "127.0.0.1");
+  await once(upstream, "listening");
+
+  gateway = spawn(process.execPath, ["src/server.mjs"], {
+    cwd: new URL("..", import.meta.url),
+    env: {
+      ...process.env,
+      NODE_ENV: "test",
+      PORT: String(gatewayPort),
+      MAP_CACHE_DIR: join(root, "cache"),
+      MAP_CACHE_MODE: "readwrite",
+      MAP_GATEWAY_ALLOW_ANONYMOUS: "true",
+      MAP_GATEWAY_UPSTREAM_ALLOWLIST: "127.0.0.1",
+      MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK: "true",
+    },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  await waitForGateway(gatewayPort, gateway);
+
+  const source = `http://127.0.0.1:${upstreamPort}/terrain/layer.json`;
+  const first = await requestGateway(gatewayPort, source);
+  assert.equal(first.body, "live-1");
+  assert.equal(first.response.headers.get("x-nodedc-map-cache"), "live-record");
+
+  upstreamState = "live-2";
+  const second = await requestGateway(gatewayPort, source);
+  assert.equal(second.body, "live-1");
+  assert.equal(["live-record", "live-cache-hit"].includes(second.response.headers.get("x-nodedc-map-cache")), true);
+  assert.equal(upstreamRequests, 1);
+  const cachedEtag = second.response.headers.get("etag");
+  assert.match(cachedEtag || "", /^(?:W\/)?"[^"]+"$/);
+  const indexPath = join(root, "cache", "index.json");
+  const indexBeforeHits = await stat(indexPath);
+  const indexRawBeforeHits = await readFile(indexPath, "utf8");
+
+  const revalidated = await requestGateway(gatewayPort, source, { headers: { "if-none-match": cachedEtag } });
+  assert.equal(revalidated.response.status, 304);
+  assert.equal(revalidated.body, "");
+  assert.equal(revalidated.response.headers.get("etag"), cachedEtag);
+  assert.equal(upstreamRequests, 1);
+
+  // Browser route revisions bypass only the browser HTTP cache. They must not
+  // create a duplicate upstream URL or TileCache entry.
+  const versioned = await requestGateway(gatewayPort, `${source}?nodedc_client_revision=2`);
+  assert.equal(versioned.body, "live-1");
+  assert.equal(versioned.response.headers.get("x-nodedc-map-cache"), "live-cache-hit");
+  assert.equal(upstreamRequests, 1);
+  for (let attempt = 0; attempt < 4; attempt += 1) {
+    const repeated = await requestGateway(gatewayPort, source);
+    assert.equal(repeated.response.headers.get("x-nodedc-map-cache"), "live-cache-hit");
+  }
+  const indexAfterHits = await stat(indexPath);
+  const indexRawAfterHits = await readFile(indexPath, "utf8");
+  assert.equal(indexAfterHits.ino, indexBeforeHits.ino);
+  assert.equal(indexAfterHits.mtimeMs, indexBeforeHits.mtimeMs);
+  assert.equal(indexRawAfterHits, indexRawBeforeHits);
+
+  // `nodedc_cache_refresh=1` is the explicit opposite of "Не
+  // перезаписывать cache": it goes live and replaces the stored object.
+  const refreshed = await requestGateway(gatewayPort, `${source}?nodedc_cache_refresh=1`);
+  assert.equal(refreshed.body, "live-2");
+  assert.equal(refreshed.response.headers.get("x-nodedc-map-cache"), "live-refresh-record");
+  assert.equal(upstreamRequests, 2);
+
+  upstreamState = "unavailable";
+  const retained = await requestGateway(gatewayPort, source);
+  assert.equal(retained.body, "live-2");
+  assert.equal(retained.response.headers.get("x-nodedc-map-cache"), "live-cache-hit");
+  assert.equal(upstreamRequests, 2);
+
+  // The second response takes the live-network-first path with a cache entry
+  // already present. The Gateway must not forward the compressed upstream
+  // Content-Length after Node has decoded the body.
+  const gzipSource = `http://127.0.0.1:${upstreamPort}/gzip/layer.json`;
+  const gzipFirst = await requestGateway(gatewayPort, gzipSource);
+  assert.equal(JSON.parse(gzipFirst.body).revision, 1);
+  gzipRevision = 2;
+  const gzipSecond = await requestGateway(gatewayPort, gzipSource);
+  assert.equal(JSON.parse(gzipSecond.body).revision, 1);
+  assert.equal(
+    ["live-record", "live-cache-hit"].includes(gzipSecond.response.headers.get("x-nodedc-map-cache")),
+    true,
+  );
+  const gzipRefresh = await requestGateway(gatewayPort, `${gzipSource}?nodedc_cache_refresh=1`);
+  assert.equal(JSON.parse(gzipRefresh.body).revision, 2);
+  // Node fetch decodes gzip while retaining the encoded upstream length. A
+  // streaming response cannot know the decoded length before completion, so
+  // it must use chunking instead of forwarding the unsafe encoded value.
+  assert.equal(gzipRefresh.response.headers.get("content-length"), null);
+  console.log("ok: TileCache hits do zero index writes, support ETag/304, and only explicit refresh replaces an object");
+} finally {
+  await stop(gateway);
+  if (upstream?.listening) {
+    upstream.close();
+    await once(upstream, "close");
+  }
+  await rm(root, { recursive: true, force: true });
+}
+
+async function requestGateway(port, source, options = {}) {
+  const response = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(source)}`, options);
+  return { response, body: await response.text() };
+}
+
+async function freePort() {
+  const server = createNetServer();
+  server.listen(0, "127.0.0.1");
+  await once(server, "listening");
+  const address = server.address();
+  assert(address && typeof address === "object");
+  const port = address.port;
+  server.close();
+  await once(server, "close");
+  return port;
+}
+
+async function waitForGateway(port, child) {
+  let output = "";
+  child.stderr.on("data", (chunk) => { output += String(chunk); });
+  for (let attempt = 0; attempt < 80; attempt += 1) {
+    if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`);
+    try {
+      const response = await fetch(`http://127.0.0.1:${port}/healthz`);
+      if (response.ok) return;
+    } catch { /* service is still starting */ }
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error(`gateway_start_timeout:${output}`);
+}
+
+async function stop(child) {
+  if (child && !child.killed) {
+    child.kill("SIGTERM");
+    await once(child, "exit").catch(() => undefined);
+  }
+}