From 7b55d887bc06d69d4fa0e019864149eeaa1b042f Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 02:23:56 +0300 Subject: [PATCH] feat(map): add cache-first Cesium gateway and AMD egress --- services/dc-amd-connector/.env.example | 3 + services/dc-amd-connector/.gitignore | 2 + services/dc-amd-connector/Dockerfile | 12 + services/dc-amd-connector/README.md | 84 ++ services/dc-amd-connector/SHA256SUMS | 10 + services/dc-amd-connector/VERSION | 1 + services/dc-amd-connector/docker-compose.yml | 47 + services/dc-amd-connector/install.ps1 | 170 +++ services/dc-amd-connector/package.json | 9 + services/dc-amd-connector/server.mjs | 130 ++ services/dc-amd-connector/uninstall.ps1 | 30 + services/dc-amd-proxy/Dockerfile | 12 + services/dc-amd-proxy/OPERATIONS.md | 770 +++++++++++ services/dc-amd-proxy/README.md | 84 ++ services/dc-amd-proxy/docker-compose.yml | 47 + services/dc-amd-proxy/package.json | 10 + .../scripts/smoke-connection-pool.mjs | 499 ++++++++ services/dc-amd-proxy/server.mjs | 987 ++++++++++++++ services/map-gateway/Dockerfile | 6 +- services/map-gateway/README.md | 49 +- services/map-gateway/docker-entrypoint.sh | 11 - services/map-gateway/package.json | 4 + services/map-gateway/runtime-entrypoint.mjs | 17 + .../scripts/smoke-admin-boundary.mjs | 366 ++++++ .../scripts/smoke-live-cache-fallback.mjs | 177 +++ .../scripts/smoke-no-browser-credentials.mjs | 117 ++ .../scripts/smoke-streaming-cache-fill.mjs | 246 ++++ services/map-gateway/src/server.mjs | 1139 +++++++++++++++-- tools/dc-amd-pair/README.txt | 11 + tools/dc-amd-pair/SHA256SUMS | 2 + tools/dc-amd-pair/pair-dc-amd-proxy.ps1 | 71 + 31 files changed, 4956 insertions(+), 167 deletions(-) create mode 100644 services/dc-amd-connector/.env.example create mode 100644 services/dc-amd-connector/.gitignore create mode 100644 services/dc-amd-connector/Dockerfile create mode 100644 services/dc-amd-connector/README.md create mode 100644 services/dc-amd-connector/SHA256SUMS create mode 100644 services/dc-amd-connector/VERSION create mode 100644 services/dc-amd-connector/docker-compose.yml create mode 100644 services/dc-amd-connector/install.ps1 create mode 100644 services/dc-amd-connector/package.json create mode 100644 services/dc-amd-connector/server.mjs create mode 100644 services/dc-amd-connector/uninstall.ps1 create mode 100644 services/dc-amd-proxy/Dockerfile create mode 100644 services/dc-amd-proxy/OPERATIONS.md create mode 100644 services/dc-amd-proxy/README.md create mode 100644 services/dc-amd-proxy/docker-compose.yml create mode 100644 services/dc-amd-proxy/package.json create mode 100644 services/dc-amd-proxy/scripts/smoke-connection-pool.mjs create mode 100644 services/dc-amd-proxy/server.mjs delete mode 100644 services/map-gateway/docker-entrypoint.sh create mode 100644 services/map-gateway/runtime-entrypoint.mjs create mode 100644 services/map-gateway/scripts/smoke-admin-boundary.mjs create mode 100644 services/map-gateway/scripts/smoke-live-cache-fallback.mjs create mode 100644 services/map-gateway/scripts/smoke-no-browser-credentials.mjs create mode 100644 services/map-gateway/scripts/smoke-streaming-cache-fill.mjs create mode 100644 tools/dc-amd-pair/README.txt create mode 100644 tools/dc-amd-pair/SHA256SUMS create mode 100644 tools/dc-amd-pair/pair-dc-amd-proxy.ps1 diff --git a/services/dc-amd-connector/.env.example b/services/dc-amd-connector/.env.example new file mode 100644 index 0000000..922b23f --- /dev/null +++ b/services/dc-amd-connector/.env.example @@ -0,0 +1,3 @@ +# Stable LAN address of the AMD Windows host. This is not the OpenVPN address. +AMD_CONNECTOR_BIND_IP=172.22.0.183 +AMD_CONNECTOR_PORT=8791 diff --git a/services/dc-amd-connector/.gitignore b/services/dc-amd-connector/.gitignore new file mode 100644 index 0000000..147af14 --- /dev/null +++ b/services/dc-amd-connector/.gitignore @@ -0,0 +1,2 @@ +.env +runtime/ diff --git a/services/dc-amd-connector/Dockerfile b/services/dc-amd-connector/Dockerfile new file mode 100644 index 0000000..d8283bd --- /dev/null +++ b/services/dc-amd-connector/Dockerfile @@ -0,0 +1,12 @@ +FROM node:20-alpine + +WORKDIR /app + +COPY package.json ./ +COPY server.mjs ./ + +USER node + +EXPOSE 8791 + +CMD ["node", "server.mjs"] diff --git a/services/dc-amd-connector/README.md b/services/dc-amd-connector/README.md new file mode 100644 index 0000000..ec07dfc --- /dev/null +++ b/services/dc-amd-connector/README.md @@ -0,0 +1,84 @@ +# DC AMD Connector + +DC AMD Connector runs on the adjacent AMD Windows machine. It is a restricted +HTTP `CONNECT` proxy for the NODE.DC map path: + +```text +NAS DC AMD Proxy -> 172.22.0.183:8791 -> AMD VPN -> Cesium / Bing +``` + +It is not a general-purpose proxy: + +- only authenticated `CONNECT` requests are accepted; +- only port `443` is accepted; +- only Cesium Ion and required Bing imagery hosts are accepted; +- ordinary HTTP proxy requests are rejected; +- the Ion bearer token stays inside the end-to-end TLS connection from NAS to + Cesium and is not processed or logged by this connector. + +The Windows host owns the VPN. The NAS uses only the stable LAN address in +`.env`, never an OpenVPN adapter address or a changing public VPN exit IP. + +## First run on the AMD machine + +1. Start Docker Desktop and wait for its engine to be running. +2. Open **PowerShell as Administrator** and run `install.ps1` from this + package folder. It verifies source checksums, copies the package to + `C:\\NODEDC\\dc-amd-connector`, creates a local random access token without + printing it, binds port `8791` only to `172.22.0.183`, scopes Windows + Firewall to NAS `172.22.0.222`, constrains the container to one CPU, 256 MB, + 64 processes and 4096 file descriptors, and starts the container. It also + adds a launcher to the current user's Windows Startup folder. At the next + user sign-in it uses the Docker Desktop CLI when available, otherwise its + installed application; the `unless-stopped` restart policy then restores + this container. + + ```powershell + powershell -NoProfile -ExecutionPolicy Bypass -File .\\install.ps1 + ``` + +3. Do not send the runtime secret in chat or put it into a source file. The + next NAS patch will provision the same value in a root-owned secret file + through an operator-only transfer. + +Docker Desktop on the WSL 2 backend is a desktop application, not an +unattended Windows boot daemon. Thus this recovery guarantee starts after the +configured Windows user signs in. Before a sign-in, the map must use its +offline/tile-cache path. No Windows route, VPN, DNS, proxy setting or other +application's traffic is modified. + +## Operations and migration + +Run these commands from an elevated PowerShell window: + +```powershell +cd C:\\NODEDC\\dc-amd-connector +docker compose ps +docker compose logs --tail 100 +docker compose restart +``` + +`docker compose down` is an intentional stop and removes the container; restore +it with `docker compose up -d`. The auto-start launcher is removed by +`uninstall.ps1`. + +To move to a different adjacent Windows host: install this same package there +with its stable LAN address (`-BindAddress`) and verify its VPN tunnel first. +It creates a fresh local access secret. Only then may the separate NAS pairing +patch be changed to the new host; never run two active pairings against the +same NAS endpoint. + +## Verification + +The connector health check uses the local runtime secret inside the container. +It is expected to become `healthy` without making any external request. The +following local check proves a permitted TLS tunnel through the current AMD VPN +without revealing the access token: + +```powershell +docker exec dc-amd-connector node -e "const fs=require('fs'),net=require('net');const t=fs.readFileSync('/run/dc-amd-secrets/connector-access','utf8').trim();const s=net.connect(8791,'127.0.0.1',()=>s.write('CONNECT api.cesium.com:443 HTTP/1.1\r\nHost: api.cesium.com:443\r\nProxy-Authorization: Bearer '+t+'\r\n\r\n'));s.once('data',d=>{const v=d.toString('ascii');console.log(v.split('\r\n')[0]);s.destroy();process.exit(v.startsWith('HTTP/1.1 200')?0:1)});s.on('error',e=>{console.error(e.message);process.exit(1)})" +``` + +Expected output is `HTTP/1.1 200 Connection Established`. If the AMD VPN is +off, this test must fail; it must never silently move live Cesium traffic to +the NAS. diff --git a/services/dc-amd-connector/SHA256SUMS b/services/dc-amd-connector/SHA256SUMS new file mode 100644 index 0000000..027e613 --- /dev/null +++ b/services/dc-amd-connector/SHA256SUMS @@ -0,0 +1,10 @@ +36283c8f926cd5a2fdfec0adda20e1d30fefb47e8fb6f4c6a41ebdfdb8034357 ./.env.example +7e4d35564197224edd4899aae9525c5b3c0c904371bb2e635dcef7950bc776cd ./.gitignore +3d7464d3a7b97b3b9be1036dcf77a9fd31cd841043a64069870e001a7f526a7b ./Dockerfile +c967a21fbb804a61e467fd207702f328ccc5ed933beaf98a476490f033f6dfa4 ./README.md +08708bd47ddf85dadda552eeb2ac1bc8f8d1e5d119305ef41bb0327b4dbac36b ./VERSION +b00bbf2ec6b56d834b04543d4e65bda7eff97296d4d255df48c28d71369a3f0e ./docker-compose.yml +608e3c75b69567376019b6f9fdf274a5cf9afecdde07348b6f02c89b688e672a ./install.ps1 +33d4c94fdec3b47bba25039306423dd9dea50d4b36014a5a34768e0a6a7ba174 ./package.json +e08d0f33b4749289a5adf897b8a4ef876d3ebad9aa377d992e9d33b1060fe698 ./server.mjs +f9bb0a9d862e9f78fd533eeb2a0750108392d73f6173aec2ccc8053120638e01 ./uninstall.ps1 diff --git a/services/dc-amd-connector/VERSION b/services/dc-amd-connector/VERSION new file mode 100644 index 0000000..6009c10 --- /dev/null +++ b/services/dc-amd-connector/VERSION @@ -0,0 +1 @@ +dc-amd-connector-20260715-004 diff --git a/services/dc-amd-connector/docker-compose.yml b/services/dc-amd-connector/docker-compose.yml new file mode 100644 index 0000000..01a77c0 --- /dev/null +++ b/services/dc-amd-connector/docker-compose.yml @@ -0,0 +1,47 @@ +services: + dc-amd-connector: + build: + context: . + dockerfile: Dockerfile + container_name: dc-amd-connector + image: nodedc/dc-amd-connector:local + restart: unless-stopped + # This is a byte-forwarding gateway, not a map renderer. These are circuit + # breakers for the shared workstation, not normal throughput limits. + cpus: "1.00" + mem_limit: 256m + pids_limit: 64 + ulimits: + nofile: + soft: 4096 + hard: 4096 + env_file: + - .env + environment: + PORT: "8791" + AMD_CONNECTOR_ACCESS_TOKEN_FILE: /run/dc-amd-secrets/connector-access + # Bind only to the stable LAN address of the AMD host, never all Windows + # interfaces and never the OpenVPN adapter address. + ports: + - "${AMD_CONNECTOR_BIND_IP:?set AMD_CONNECTOR_BIND_IP}:${AMD_CONNECTOR_PORT:-8791}:8791" + volumes: + - type: bind + source: ./runtime/connector-access + target: /run/dc-amd-secrets/connector-access + read_only: true + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD-SHELL + - > + node -e "const fs=require('fs'),http=require('http');const t=fs.readFileSync('/run/dc-amd-secrets/connector-access','utf8').trim();const r=http.get({host:'127.0.0.1',port:8791,path:'/healthz',headers:{'proxy-authorization':'Bearer '+t}},x=>process.exit(x.statusCode===200?0:1));r.on('error',()=>process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s diff --git a/services/dc-amd-connector/install.ps1 b/services/dc-amd-connector/install.ps1 new file mode 100644 index 0000000..80fbeb1 --- /dev/null +++ b/services/dc-amd-connector/install.ps1 @@ -0,0 +1,170 @@ +[CmdletBinding()] +param( + [string]$Destination = 'C:\NODEDC\dc-amd-connector', + [string]$NasAddress = '172.22.0.222', + [string]$BindAddress = '172.22.0.183', + [int]$Port = 8791, + [bool]$EnableAutoStart = $true +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Require-Administrator { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = New-Object Security.Principal.WindowsPrincipal($identity) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Run this installer from an elevated PowerShell window.' + } +} + +function Test-PackageChecksums([string]$Root) { + $manifest = Join-Path $Root 'SHA256SUMS' + if (-not (Test-Path -LiteralPath $manifest -PathType Leaf)) { + throw "Checksum manifest not found: $manifest" + } + + foreach ($line in Get-Content -LiteralPath $manifest) { + if ([string]::IsNullOrWhiteSpace($line)) { continue } + $parts = $line -split '\s{2,}', 2 + if ($parts.Count -ne 2 -or $parts[0] -notmatch '^[a-f0-9]{64}$') { + throw "Invalid checksum line: $line" + } + $relative = $parts[1] -replace '^\./', '' + $file = Join-Path $Root $relative + $actual = (Get-FileHash -LiteralPath $file -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actual -ne $parts[0]) { throw "Checksum mismatch: $relative" } + } +} + +function Get-DockerDesktopStartCommand([string]$DockerCliPath) { + & $DockerCliPath desktop start --help 2>$null | Out-Null + if ($LASTEXITCODE -eq 0) { + return "`"$DockerCliPath`" desktop start --detach" + } + + $dockerRoot = Split-Path (Split-Path (Split-Path $DockerCliPath -Parent) -Parent) -Parent + $candidates = @( + (Join-Path $dockerRoot 'Docker Desktop.exe'), + (Join-Path $env:ProgramFiles 'Docker\Docker\Docker Desktop.exe'), + (Join-Path $env:LOCALAPPDATA 'Programs\Docker\Docker\Docker Desktop.exe') + ) | Select-Object -Unique + $desktopExe = $candidates | Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } | Select-Object -First 1 + if (-not $desktopExe) { + throw 'Unable to find Docker Desktop start command or Docker Desktop.exe for restart recovery.' + } + return "start `"`" `"$desktopExe`"" +} + +function New-DockerDesktopAutoStart([string]$StartCommand) { + $startup = [Environment]::GetFolderPath([Environment+SpecialFolder]::Startup) + if ([string]::IsNullOrWhiteSpace($startup)) { throw 'Unable to resolve the current user Startup folder.' } + + $launcher = Join-Path $startup 'NODE.DC DC AMD Connector - Docker Desktop.cmd' + if (Test-Path -LiteralPath $launcher) { + throw "Docker Desktop auto-start launcher already exists: $launcher" + } + + @( + '@echo off', + 'rem NODE.DC DC AMD Connector - starts Docker Desktop after this user signs in.', + $StartCommand + ) | Set-Content -LiteralPath $launcher -Encoding ascii + return $launcher +} + +Require-Administrator +$source = Split-Path -Parent $PSCommandPath +if (-not (Test-Path -LiteralPath (Join-Path $source 'VERSION') -PathType Leaf)) { + throw 'Run install.ps1 from the DC AMD Connector package folder.' +} +Test-PackageChecksums $source + +if (Test-Path -LiteralPath $Destination) { + throw "Destination already exists; refusing to overwrite: $Destination" +} + +$null = [Net.IPAddress]::Parse($NasAddress) +$null = [Net.IPAddress]::Parse($BindAddress) +if ($Port -lt 1 -or $Port -gt 65535) { + throw "Port must be between 1 and 65535: $Port" +} + +$dockerCli = (Get-Command docker -CommandType Application -ErrorAction Stop | Select-Object -First 1).Path +if ([string]::IsNullOrWhiteSpace($dockerCli) -or -not (Test-Path -LiteralPath $dockerCli -PathType Leaf)) { + throw 'Docker CLI executable could not be resolved.' +} +& $dockerCli version --format '{{.Server.Os}}/{{.Server.Arch}} {{.Server.Version}}' | Out-Null +if ($LASTEXITCODE -ne 0) { + throw 'Docker Desktop engine is not available. Start it and wait for Engine running.' +} +$dockerDesktopStartCommand = Get-DockerDesktopStartCommand $dockerCli + +$destinationCreated = $false +$firewallCreated = $false +$autoStartLauncher = $null +$parent = Split-Path -Parent $Destination +try { + New-Item -ItemType Directory -Force $parent | Out-Null + Copy-Item -LiteralPath $source -Destination $Destination -Recurse + $destinationCreated = $true + Test-PackageChecksums $Destination + Set-Location $Destination + + New-Item -ItemType Directory -Force runtime | Out-Null + $tokenPath = Join-Path $Destination 'runtime\connector-access' + if (Test-Path -LiteralPath $tokenPath) { throw "Runtime token already exists: $tokenPath" } + $bytes = New-Object byte[] 48 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { $rng.GetBytes($bytes) } finally { $rng.Dispose() } + $token = [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') + [IO.File]::WriteAllText($tokenPath, $token, [Text.UTF8Encoding]::new($false)) + + Copy-Item -LiteralPath (Join-Path $Destination '.env.example') -Destination (Join-Path $Destination '.env') + @( + "AMD_CONNECTOR_BIND_IP=$BindAddress", + "AMD_CONNECTOR_PORT=$Port" + ) | Set-Content -LiteralPath (Join-Path $Destination '.env') -Encoding ascii + + $ruleName = 'NODE.DC DC AMD Connector (NAS only)' + if (Get-NetFirewallRule -DisplayName $ruleName -ErrorAction SilentlyContinue) { + throw "Firewall rule already exists; refusing to replace: $ruleName" + } + New-NetFirewallRule -DisplayName $ruleName -Direction Inbound -Action Allow -Protocol TCP -Profile Any -LocalAddress $BindAddress -LocalPort $Port -RemoteAddress $NasAddress | Out-Null + $firewallCreated = $true + + if ($EnableAutoStart) { + $autoStartLauncher = New-DockerDesktopAutoStart $dockerDesktopStartCommand + } + + & $dockerCli compose up -d --build + $state = '' + foreach ($attempt in 1..30) { + $state = & $dockerCli inspect dc-amd-connector --format '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}}' + if ($LASTEXITCODE -ne 0) { throw 'Container inspection failed after Docker Compose apply.' } + if ($state -eq 'running health=healthy') { break } + Start-Sleep -Seconds 2 + } + if ($state -ne 'running health=healthy') { + throw "Connector did not become healthy: $state" + } + + Write-Host "DC AMD Connector installed: $state" + Write-Host "Bound to $BindAddress`:$Port; inbound firewall scope is NAS $NasAddress." + if ($autoStartLauncher) { + Write-Host "Docker Desktop will start at the next sign-in through: $autoStartLauncher" + } + Write-Host 'The connector access token remains only in runtime\connector-access. Do not print or send it in chat.' +} +catch { + if ($autoStartLauncher -and (Test-Path -LiteralPath $autoStartLauncher)) { + Remove-Item -LiteralPath $autoStartLauncher -Force -ErrorAction SilentlyContinue + } + if ($firewallCreated) { + Remove-NetFirewallRule -DisplayName 'NODE.DC DC AMD Connector (NAS only)' -ErrorAction SilentlyContinue + } + if ($destinationCreated -and (Test-Path -LiteralPath $Destination)) { + Remove-Item -LiteralPath $Destination -Recurse -Force -ErrorAction SilentlyContinue + } + throw +} diff --git a/services/dc-amd-connector/package.json b/services/dc-amd-connector/package.json new file mode 100644 index 0000000..20bb482 --- /dev/null +++ b/services/dc-amd-connector/package.json @@ -0,0 +1,9 @@ +{ + "name": "dc-amd-connector", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node server.mjs" + } +} diff --git a/services/dc-amd-connector/server.mjs b/services/dc-amd-connector/server.mjs new file mode 100644 index 0000000..040791d --- /dev/null +++ b/services/dc-amd-connector/server.mjs @@ -0,0 +1,130 @@ +import { createServer } from "node:http"; +import { readFile } from "node:fs/promises"; +import { connect } from "node:net"; +import { timingSafeEqual } from "node:crypto"; + +const allowedHosts = new Set([ + "api.cesium.com", + "assets.ion.cesium.com", + "dev.virtualearth.net", + "ecn.t0.tiles.virtualearth.net", + "ecn.t1.tiles.virtualearth.net", + "ecn.t2.tiles.virtualearth.net", + "ecn.t3.tiles.virtualearth.net", +]); +const config = await readConfig(); + +const server = createServer((request, response) => { + if (!isAuthorized(request.headers["proxy-authorization"])) return writeJson(response, 401, { ok: false, error: "connector_unauthorized" }, { "proxy-authenticate": "Bearer" }); + if (request.method === "GET" && request.url === "/healthz") return writeJson(response, 200, { ok: true, service: "dc-amd-connector", mode: "restricted-connect", allowedHosts: allowedHosts.size }); + return writeJson(response, 405, { ok: false, error: "connect_only" }); +}); + +server.on("connect", (request, clientSocket, head) => { + if (!isAuthorized(request.headers["proxy-authorization"])) return rejectTunnel(clientSocket, 407, "Proxy Authentication Required", { "Proxy-Authenticate": "Bearer" }); + + let target; + try { + target = parseConnectTarget(request.url || ""); + } catch (error) { + log("warn", "connect_rejected", { reason: error.message }); + return rejectTunnel(clientSocket, 403, "Forbidden"); + } + + const upstream = connect({ host: target.host, port: target.port }); + let settled = false; + const timeout = setTimeout(() => upstream.destroy(new Error("upstream_connect_timeout")), config.connectTimeoutMs); + clientSocket.setTimeout(config.idleTimeoutMs, () => clientSocket.destroy()); + upstream.setTimeout(config.idleTimeoutMs, () => upstream.destroy()); + + upstream.once("connect", () => { + settled = true; + clearTimeout(timeout); + clientSocket.write("HTTP/1.1 200 Connection Established\r\nProxy-Agent: dc-amd-connector\r\n\r\n"); + if (head?.length) upstream.write(head); + clientSocket.pipe(upstream); + upstream.pipe(clientSocket); + log("info", "connect_established", { host: target.host, port: target.port }); + }); + upstream.once("error", (error) => { + clearTimeout(timeout); + if (!settled) { + log("warn", "connect_failed", { host: target.host, port: target.port, reason: sanitizeError(error) }); + rejectTunnel(clientSocket, 502, "Bad Gateway"); + } + }); + clientSocket.once("error", () => upstream.destroy()); + clientSocket.once("close", () => upstream.destroy()); + upstream.once("close", () => clientSocket.destroy()); +}); + +server.on("clientError", (_error, socket) => rejectTunnel(socket, 400, "Bad Request")); +server.listen(config.port, "0.0.0.0", () => log("info", "connector_started", { port: config.port, allowedHosts: allowedHosts.size })); + +for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => server.close(() => process.exit(0))); + +async function readConfig() { + const port = parsePort(process.env.PORT, 8791); + const tokenFile = String(process.env.AMD_CONNECTOR_ACCESS_TOKEN_FILE || "/run/dc-amd-secrets/connector-access").trim(); + const token = String(await readFile(tokenFile, "utf8")).trim(); + if (!/^[A-Za-z0-9_-]{48,256}$/.test(token)) throw new Error("connector_access_token_invalid"); + return { + port, + token: Buffer.from(token, "utf8"), + connectTimeoutMs: parseDuration(process.env.AMD_CONNECTOR_CONNECT_TIMEOUT_SECONDS, 20), + idleTimeoutMs: parseDuration(process.env.AMD_CONNECTOR_IDLE_TIMEOUT_SECONDS, 90), + }; +} + +function parsePort(raw, fallback) { + const value = Number(String(raw || fallback).trim()); + if (!Number.isInteger(value) || value < 1024 || value > 65535) throw new Error("connector_port_invalid"); + return value; +} + +function parseDuration(raw, fallbackSeconds) { + const seconds = Number(String(raw || fallbackSeconds).trim()); + if (!Number.isInteger(seconds) || seconds < 1 || seconds > 600) throw new Error("connector_timeout_invalid"); + return seconds * 1000; +} + +function isAuthorized(rawHeader) { + const match = /^Bearer\s+([A-Za-z0-9_-]{48,256})$/i.exec(String(rawHeader || "").trim()); + if (!match) return false; + const candidate = Buffer.from(match[1], "utf8"); + return candidate.length === config.token.length && timingSafeEqual(candidate, config.token); +} + +function parseConnectTarget(raw) { + const match = /^([A-Za-z0-9.-]{1,253}):(443)$/.exec(String(raw || "").trim()); + if (!match) throw new Error("connect_target_invalid"); + const host = match[1].toLowerCase(); + if (!allowedHosts.has(host)) throw new Error("connect_target_not_allowed"); + return { host, port: Number(match[2]) }; +} + +function rejectTunnel(socket, status, message, headers = {}) { + if (!socket || socket.destroyed) return; + const extra = Object.entries(headers).map(([name, value]) => `${name}: ${value}\r\n`).join(""); + socket.end(`HTTP/1.1 ${status} ${message}\r\n${extra}Connection: close\r\nContent-Length: 0\r\n\r\n`); +} + +function writeJson(response, status, body, headers = {}) { + const payload = JSON.stringify(body); + response.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(payload), + "cache-control": "no-store", + ...headers, + }); + response.end(payload); +} + +function sanitizeError(error) { + const message = String(error?.message || "connector_error"); + return message.replace(/[^A-Za-z0-9_.:-]/g, "_").slice(0, 120); +} + +function log(level, event, fields = {}) { + console.log(JSON.stringify({ ts: new Date().toISOString(), level, event, ...fields })); +} diff --git a/services/dc-amd-connector/uninstall.ps1 b/services/dc-amd-connector/uninstall.ps1 new file mode 100644 index 0000000..5cd347a --- /dev/null +++ b/services/dc-amd-connector/uninstall.ps1 @@ -0,0 +1,30 @@ +[CmdletBinding()] +param( + [string]$Destination = 'C:\NODEDC\dc-amd-connector' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Require-Administrator { + $identity = [Security.Principal.WindowsIdentity]::GetCurrent() + $principal = New-Object Security.Principal.WindowsPrincipal($identity) + if (-not $principal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { + throw 'Run this uninstaller from an elevated PowerShell window.' + } +} + +Require-Administrator +$docker = Get-Command docker -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 +if ($docker -and (Test-Path -LiteralPath (Join-Path $Destination 'docker-compose.yml'))) { + Push-Location $Destination + try { & $docker.Path compose down --remove-orphans } finally { Pop-Location } +} + +Remove-NetFirewallRule -DisplayName 'NODE.DC DC AMD Connector (NAS only)' -ErrorAction SilentlyContinue +$startup = [Environment]::GetFolderPath([Environment+SpecialFolder]::Startup) +$launcher = Join-Path $startup 'NODE.DC DC AMD Connector - Docker Desktop.cmd' +Remove-Item -LiteralPath $launcher -Force -ErrorAction SilentlyContinue +Remove-Item -LiteralPath $Destination -Recurse -Force -ErrorAction SilentlyContinue + +Write-Host 'DC AMD Connector removed. Docker Desktop and unrelated containers were not changed.' diff --git a/services/dc-amd-proxy/Dockerfile b/services/dc-amd-proxy/Dockerfile new file mode 100644 index 0000000..9fc5c3a --- /dev/null +++ b/services/dc-amd-proxy/Dockerfile @@ -0,0 +1,12 @@ +FROM node:20-alpine + +WORKDIR /app + +COPY package.json ./ +COPY server.mjs ./ + +USER node + +EXPOSE 8790 + +CMD ["node", "server.mjs"] diff --git a/services/dc-amd-proxy/OPERATIONS.md b/services/dc-amd-proxy/OPERATIONS.md new file mode 100644 index 0000000..ad46be5 --- /dev/null +++ b/services/dc-amd-proxy/OPERATIONS.md @@ -0,0 +1,770 @@ +# DC AMD Proxy — production operations and recovery canon + +This document is the reproducible operations contract for the restricted +Cesium/Bing egress path through the neighbouring Windows AMD workstation. It +describes the implementation that is in source now. It is not a proposal for +turning the NAS or the workstation into a general-purpose proxy. + +The two non-negotiable boundaries are: + +1. the NAS keeps its own routes, DNS, VPN, Tailscale and Internet connectivity + unchanged; +2. only approved Cesium/Bing HTTPS traffic may leave through the AMD host. + +Do not solve an incident by adding a system proxy, default route, transparent +NAT, browser-visible token, permissive CONNECT endpoint, `0.0.0.0` Windows +bind, or direct NAS provider fallback. + +## 1. Canonical architecture + +```text +Browser + -> Foundry BFF + -> Platform Map Gateway + |-- warm object: NAS live/offline cache -> browser + `-- cold/refresh object: + HTTP + x-proxy-token + -> NAS : dc-amd-proxy + -> authenticated HTTP CONNECT + -> AMD : dc-amd-connector + -> AMD/Docker DNS + -> Windows VPN/default egress route + -> approved Cesium/Bing host:443 + <- end-to-end provider TLS is terminated by dc-amd-proxy + <- streamed provider response + -> browser and, when enabled, atomic NAS cache fill +``` + +The AMD connector is a byte-forwarding CONNECT boundary. It does not parse an +Ion bearer and cannot read the provider TLS stream. `dc-amd-proxy` establishes +TLS through that tunnel, validates the provider certificate and SNI, follows +only approved redirects, and returns the response to Map Gateway. + +This distinction is operationally important: + +- a **warm cache hit does not traverse AMD, VPN or Cesium**; +- a **cold miss or refresh does traverse AMD and depends on the workstation's + Docker Desktop, DNS, routing and VPN**; +- objects already cached on NAS must remain usable when AMD or its VPN is + unavailable; +- `dc-amd-proxy` is not in the browser-to-NAS cache read path. + +### Current components and ownership + +| Component | Runtime | Canonical state | Responsibility | +| --- | --- | --- | --- | +| Map Gateway | Synology Docker | `/volume1/docker/nodedc-platform/platform/services/map-gateway` | Provider credential scope, cache-first reads, streaming cache fill and private egress calls | +| `dc-amd-proxy` | Synology Docker | `/volume1/docker/dc-amd-proxy` | Request authorization, target allowlist, CONNECT/TLS pool, redirect/retry policy and telemetry | +| Proxy pair state | Synology bind mount | `/volume1/docker/dc-amd-proxy/runtime/connector-access` | Paired connector credential; persistent across container recreation | +| Map egress credential | Synology root-owned file | `/volume1/docker/nodedc-platform/secrets/map-egress-proxy-token` | Authenticates Map Gateway to `dc-amd-proxy` | +| `dc-amd-connector` | AMD Windows Docker Desktop | `C:\NODEDC\dc-amd-connector` by default | Restricted authenticated CONNECT to approved hosts on port 443 | +| Connector credential | AMD local file | `C:\NODEDC\dc-amd-connector\runtime\connector-access` | Generated locally once; never printed or stored in source | +| Docker Desktop launcher | Current Windows user's Startup folder | `NODE.DC DC AMD Connector - Docker Desktop.cmd` | Starts Docker Desktop after that user signs in | + +Current source defaults are NAS listener port `8790`, AMD connector port +`8791`, eight active sockets and four idle sockets per origin. IP addresses +are deployment values and must be treated as configuration, not universal +constants. + +## 2. Trust, secrets and traffic boundaries + +There are three different credentials. They are not interchangeable. + +1. The Cesium Ion credential belongs to Map Gateway provider configuration. + It must not be placed in Foundry browser code, an artifact or this proxy's + Compose file. +2. The Map egress credential authenticates Map Gateway to `dc-amd-proxy` using + `x-proxy-token`. The canonical runner synchronizes it into the root-owned + file mounted read-only into the two services. +3. The connector credential authenticates `dc-amd-proxy` to the Windows + connector using `Proxy-Authorization: Bearer ...`. The Windows installer + generates it locally; the one-time pair request stores the same value in + the NAS runtime bind mount. + +No command in this runbook prints, copies to chat, or accepts a secret as a +command-line argument. + +### Current runner dependency that must not be mistaken for a traffic route + +The deploy runner currently seeds the shared Map egress credential from the +root-owned `PROXY_TOKEN` in `/volume1/docker/proxy-contur/.env`. Therefore a +valid, safely permissioned historical `proxy-contur` environment file is a +**deploy-time precondition** for `dc-amd-proxy` and Map Gateway applies. + +This does **not** mean Cesium traffic passes through `proxy-contur`. Runtime +traffic goes directly from Map Gateway to `dc-amd-proxy`, then to the AMD +connector. Removing this legacy secret-seeding dependency is an explicit +evolution item later in this document. + +### NAS network boundary + +`dc-amd-proxy` uses `network_mode: host` so the one-time pair endpoint can see +the real AMD source address. Synology's published-port bridge path would hide +that source behind Docker NAT. Host mode does not make the service broad: + +- the process binds only `:`; +- `/api/pair` accepts only `` and only while unpaired; +- `/proxy/cesium/fetch` accepts only `GET`/`HEAD`, requires the private Map + egress credential and validates the complete target URL; +- `/healthz` and `/status` expose safe state/counters but no credentials or + target URLs; +- the container is read-only, drops all capabilities, enables + `no-new-privileges`, and has only a small no-exec tmpfs plus the two explicit + bind mounts. + +The current Map Gateway URL is a NAS-LAN address, not a public hostname and +not a Docker service discovery name. + +### Windows network boundary + +The connector publishes only `:`. The installer +creates one inbound Windows Firewall rule with all of these restrictions: + +- TCP only; +- local address ``; +- local port ``; +- remote address ``; +- no bind to the VPN adapter and no bind to all interfaces. + +The connector accepts only authenticated `CONNECT :443`. +Ordinary forward-proxy HTTP methods, other ports, IP literals and arbitrary +hosts are rejected. Its container is read-only, drops capabilities and is +bounded to one CPU, 256 MB memory, 64 processes and 4096 file descriptors. +These are circuit breakers for a shared production workstation, not bandwidth +throttles. + +The installer does not change the Windows default route, DNS, system proxy, +VPN configuration or unrelated firewall rules. The connector's DNS lookup and +TCP connection occur inside Docker Desktop on the AMD host and therefore use +the egress path exposed to Docker by Windows/VPN. + +### Exact provider allowlist + +Both NAS proxy and Windows connector currently allow only: + +- `api.cesium.com` +- `assets.ion.cesium.com` +- `dev.virtualearth.net` +- `ecn.t0.tiles.virtualearth.net` +- `ecn.t1.tiles.virtualearth.net` +- `ecn.t2.tiles.virtualearth.net` +- `ecn.t3.tiles.virtualearth.net` + +Adding a provider hostname requires one reviewed change to both allowlists, +updated smoke coverage, a new Windows connector package, a new NAS app-overlay +artifact and the normal plan/apply acceptance. Never add a wildcard such as +`*.cesium.com` or `*.virtualearth.net`. + +## 3. Request lifecycle + +For a cache miss, the sequence is: + +1. Map Gateway checks the NAS cache before acquiring provider credentials or + contacting egress. +2. Map Gateway calls + `http://:/proxy/cesium/fetch?url=` + and authenticates with the private Map egress credential. +3. `dc-amd-proxy` validates method, URL scheme, host, port and credentials. +4. A per-origin HTTP/1.1 keep-alive agent either reuses an existing TLS tunnel, + opens a new one, or queues behind the bounded active-socket limit. +5. For a new connection, the NAS opens an authenticated CONNECT request to + `:`. +6. The connector validates its credential, exact hostname and port 443, then + resolves/connects through the AMD network/VPN. +7. The NAS completes TLS over the byte tunnel and validates the provider + certificate. +8. Response headers and body stream back to Map Gateway. Client cancellation + propagates through queue, CONNECT, TLS, header wait, redirect drain, retry + and response streaming. +9. Map Gateway can stream the first bytes to the caller while its independent + branch completes an atomic cache fill. A failed/partial fill must not become + a valid cache object. + +Provider headers forwarded upstream are deliberately narrow: `Accept`, +`Range`, `If-None-Match`, `If-Modified-Since`, scoped provider authorization +and the Map referer. `Accept-Encoding` is set to `identity`. Hop-by-hop headers +are not forwarded back downstream. + +### Credential scope across redirects + +- same-origin redirects retain the provider `Authorization` header; +- cross-origin redirects remove `Authorization` before the next request; +- every redirect target is parsed again through the exact HTTPS/443/host + allowlist; +- redirect bodies are drained with abort and idle-timeout protection; +- the chain is bounded and terminates with + `amd_upstream_redirect_limit` rather than looping indefinitely. + +This is why the Ion API bearer cannot leak when the API redirects to the +assets or Bing origin. + +### Pool, retry and timeout policy + +| Control | Current default | Meaning | +| --- | ---: | --- | +| Active sockets | 8 per origin | Maximum concurrent provider tunnels for each approved origin | +| Idle sockets | 4 per origin | Reusable warm TLS tunnels retained for each origin | +| TCP keep-alive initial delay | 30 seconds | Socket keep-alive probe setting; actual idle lifetime is also controlled by provider/server socket closure | +| Connector/TLS timeout | 20 seconds | Bounds NAS-to-AMD CONNECT and provider TLS setup | +| Upstream header timeout | 30 seconds | Bounds one attempt until provider response headers | +| Response/redirect body idle timeout | 30 seconds | Resets on body progress; does not cap a healthy long stream | +| Slow request threshold | 2 seconds | Emits safe slow-request telemetry; does not cancel the request | +| Windows connector upstream connect timeout | 20 seconds | Bounds AMD-to-provider TCP connect | +| Windows connector tunnel idle timeout | 90 seconds | Closes a tunnel with no traffic | + +The NAS duration values accept 1–120 seconds and pool sizes accept 1–32. Do +not increase them to hide a route/VPN problem: first interpret queue, +connection, TTFB and body timings separately. + +There is at most one transport retry. It is allowed only when all conditions +are true: + +- method is idempotent `GET` or `HEAD`; +- the failed attempt used a previously reused keep-alive socket; +- failure is `ECONNRESET`, `EPIPE`, `ETIMEDOUT` or `ECONNABORTED`; +- the request has not already retried and the client has not aborted. + +A failure on a newly opened socket is not blindly replayed. A retry may open a +replacement socket after the stale reused socket is discarded. Both attempts +are included in telemetry and terminal accounting. + +### Runtime configuration reference + +All changes to these values require a reviewed Compose/source artifact; do not +inject an ad-hoc live container environment. + +| NAS proxy variable | Default | Contract | +| --- | --- | --- | +| `PORT` | `8790` | Internal/listener port | +| `DC_AMD_PROXY_BIND_ADDRESS` | deployment NAS LAN IP | Exact listener address; IPv4 only | +| `DC_AMD_CONNECTOR_HOST` | deployment AMD LAN IP | Stable connector LAN address; IPv4 only | +| `DC_AMD_CONNECTOR_PORT` | `8791` | Windows published connector port | +| `DC_AMD_PAIR_ALLOWED_SOURCE` | deployment AMD LAN IP | Only source allowed to perform first pair | +| `DC_AMD_CONNECTOR_TOKEN_FILE` | `/var/lib/dc-amd-proxy/connector-access` | Persistent paired connector state | +| `DC_AMD_MAP_EGRESS_TOKEN_FILE` | `/run/nodedc-secrets/map-egress-proxy-token` | Read-only Map Gateway authentication file | +| `DC_AMD_CONNECT_TIMEOUT_SECONDS` | `20` | CONNECT and TLS setup timeout | +| `DC_AMD_UPSTREAM_TIMEOUT_SECONDS` | `30` | Per-attempt response-header timeout | +| `DC_AMD_BODY_IDLE_TIMEOUT_SECONDS` | `30` | Progress-based response/redirect body timeout | +| `DC_AMD_POOL_MAX_SOCKETS` | `8` | Active sockets per origin | +| `DC_AMD_POOL_MAX_FREE_SOCKETS` | `4` | Idle sockets per origin | +| `DC_AMD_SLOW_REQUEST_SECONDS` | `2` | Safe slow-event threshold | + +The Windows Compose `.env` contains only `AMD_CONNECTOR_BIND_IP` and +`AMD_CONNECTOR_PORT`. The connector server also supports bounded +`AMD_CONNECTOR_CONNECT_TIMEOUT_SECONDS` (default 20) and +`AMD_CONNECTOR_IDLE_TIMEOUT_SECONDS` (default 90); changing them should be a +versioned connector-package change. + +## 4. Reproducible initial installation + +Replace every angle-bracket placeholder before executing a command. Do not +paste a credential into any placeholder. + +### 4.1 Preconditions + +- AMD workstation has a stable LAN address reserved in DHCP or statically + configured. +- NAS and AMD can reach each other on the trusted LAN. +- Docker Desktop is installed and its Linux engine is running. +- The intended Windows user can sign in after a reboot. +- The VPN is connected and exposes its route to Docker Desktop workloads. +- The reviewed Windows connector package contains a valid `SHA256SUMS`. +- The canonical NAS runner is installed at + `/usr/local/sbin/nodedc-deploy`. +- No other host is using the selected connector LAN address/port. + +### 4.2 Install the Windows connector + +Extract the reviewed connector package to a temporary local folder. Open +PowerShell **as Administrator**, enter that folder and run: + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File ".\install.ps1" ` + -Destination "C:\NODEDC\dc-amd-connector" ` + -NasAddress "" ` + -BindAddress "" ` + -Port ` + -EnableAutoStart $true +``` + +The installer is fail-closed. It verifies `SHA256SUMS`, refuses to overwrite an +existing destination or firewall rule, generates a random connector credential +without printing it, builds/starts the container, waits for `healthy`, creates +the NAS-only firewall rule and writes the current-user Docker Desktop Startup +launcher. On failure it removes only the state it created. + +Verify locally without displaying the credential: + +```powershell +Set-Location "C:\NODEDC\dc-amd-connector" +docker compose ps +docker inspect dc-amd-connector --format '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}} restart={{.HostConfig.RestartPolicy.Name}}' +docker compose logs --tail 100 +``` + +Expected state is `running health=healthy` and restart policy +`unless-stopped`. + +This safe CONNECT probe reads the credential only inside the container and +prints only the HTTP status line: + +```powershell +docker exec dc-amd-connector node -e "const fs=require('fs'),net=require('net');const t=fs.readFileSync('/run/dc-amd-secrets/connector-access','utf8').trim();const s=net.connect(8791,'127.0.0.1',()=>s.write('CONNECT api.cesium.com:443 HTTP/1.1\r\nHost: api.cesium.com:443\r\nProxy-Authorization: Bearer '+t+'\r\n\r\n'));s.once('data',d=>{const v=d.toString('ascii');console.log(v.split('\r\n')[0]);s.destroy();process.exit(v.startsWith('HTTP/1.1 200')?0:1)});s.on('error',e=>{console.error(e.message);process.exit(1)})" +``` + +Expected output is `HTTP/1.1 200 Connection Established`. This proves that the +connector can currently reach the API host; it does not validate an Ion token +or prove the VPN exit country. + +### 4.3 Build and transfer the NAS artifact + +From a clean `platform` repository checkout on the trusted build Mac: + +```bash +node infra/deploy-runner/build-dc-amd-proxy-artifact.mjs \ + dc-amd-proxy--- + +cd infra/deploy-artifacts +shasum -a 256 nodedc-dc-amd-proxy---.tgz \ + > nodedc-dc-amd-proxy---.tgz.sha256 +``` + +Copy exactly the `.tgz` and `.tgz.sha256` through SMB into: + +```text +/volume1/docker/nodedc-deploy/inbox +``` + +The builder includes only `Dockerfile`, `README.md`, `docker-compose.yml`, +`package.json` and `server.mjs`. Runtime, `.env`, credentials, logs, Windows +packages and this operations document are not deploy payload members. + +### 4.4 Canonical NAS plan/apply + +On Synology, from the canonical inbox: + +```bash +cd /volume1/docker/nodedc-deploy/inbox + +sha256sum -c \ + nodedc-dc-amd-proxy---.tgz.sha256 + +sudo /usr/local/sbin/nodedc-deploy verify-install + +sudo /usr/local/sbin/nodedc-deploy plan \ + /volume1/docker/nodedc-deploy/inbox/nodedc-dc-amd-proxy---.tgz +``` + +Accept the plan only when all are true: + +- `component=dc-amd-proxy` +- `type=app-overlay` +- `payload_root=/volume1/docker/dc-amd-proxy` +- `services=dc-amd-proxy` +- `runtime_secret=runner-synced:.../map-egress-proxy-token` +- `runtime_state=runner-prepared:/volume1/docker/dc-amd-proxy/runtime` +- `state=new` +- the file list is exactly the five allowlisted source files. + +Then apply the exact planned filename; never use “latest” discovery: + +```bash +sudo /usr/local/sbin/nodedc-deploy apply \ + /volume1/docker/nodedc-deploy/inbox/nodedc-dc-amd-proxy---.tgz +``` + +The runner preserves/prepares the private pair-state directory, synchronizes +the Map egress credential, creates a source backup, builds with Compose, +force-recreates only `dc-amd-proxy`, waits for container health, moves the +artifact to `applied`, and records the result. A new service should report +`state=awaiting_pair`. + +### 4.5 Pair Windows to NAS + +Pair only after NAS status says `awaiting_pair`. Place the reviewed pairing +tool on the AMD machine and run it from an elevated PowerShell window: + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass ` + -File ".\pair-dc-amd-proxy.ps1" ` + -NasAddress "" ` + -Port +``` + +Expected output: + +```text +AMD connector paired with NAS. No secret value was displayed. +``` + +The pair call is a one-time HTTP LAN handoff with the Windows proxy explicitly +disabled. Its security boundary is the trusted LAN plus exact AMD source IP. +After the first successful pair, later attempts return `409 already_paired`. + +Verify on NAS: + +```bash +curl -fsS http://:/status + +sudo /usr/local/bin/docker inspect dc-amd-proxy \ + --format 'status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}} network={{.HostConfig.NetworkMode}} read_only={{.HostConfig.ReadonlyRootfs}} restart={{.HostConfig.RestartPolicy.Name}}' +``` + +Expected properties are `state=paired`, `forwarding=amd_connector_only`, +`directEgress=false`, container `healthy`, network `host`, read-only `true`, +and restart `unless-stopped`. + +Finally verify one real Map Gateway request using the product health workflow. +Do not construct a shell command that reads or prints either private runtime +credential. + +For a first installation, switch Map Gateway to +`http://:` only through a separate reviewed +Platform artifact and only after the proxy is paired and its tunnel test +passes. This separation keeps a failed connector bootstrap from breaking an +already-working provider route. + +## 5. Restart and reboot behavior + +### Synology + +The NAS container has `restart: unless-stopped`. A Docker daemon or NAS reboot +restarts it. Pair state survives because `./runtime` is a bind mount outside +the container. The Map egress credential is a separate root-owned bind mount. + +If an operator intentionally runs `docker compose down`, the container is +removed and restart policy cannot recreate it. Restore it from the canonical +source directory: + +```bash +cd /volume1/docker/dc-amd-proxy +sudo /usr/local/bin/docker compose -p dc-amd-proxy up -d --build --no-deps dc-amd-proxy +``` + +Use that command only for recovery of the already-deployed version. Source +changes still require a runner artifact. + +### Windows AMD host + +Docker Desktop on the WSL 2 backend is not guaranteed to be an unattended +Windows boot service. The installed launcher starts it when the configured +Windows user signs in. After Docker Engine starts, `restart: unless-stopped` +restores the existing connector container. + +Therefore the actual recovery sequence is: + +1. Windows boots; +2. the configured user signs in; +3. Startup launches Docker Desktop; +4. Docker Engine becomes ready; +5. the existing connector container restarts; +6. the VPN must be connected and usable from Docker; +7. cold Cesium requests recover. + +Before user sign-in, or while Docker/VPN is unavailable, live misses fail and +Map Gateway must rely on existing NAS cache. If the connector container was +removed with `docker compose down`, restart policy alone cannot recreate it; +run from elevated PowerShell: + +```powershell +Set-Location "C:\NODEDC\dc-amd-connector" +docker compose up -d --build +``` + +## 6. Routine operations and observability + +### NAS commands + +```bash +curl -fsS http://:/status + +sudo /usr/local/bin/docker inspect dc-amd-proxy \ + --format 'status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}} restart={{.HostConfig.RestartPolicy.Name}}' + +sudo /usr/local/bin/docker logs --tail 200 dc-amd-proxy + +cd /volume1/docker/dc-amd-proxy +sudo /usr/local/bin/docker compose -p dc-amd-proxy ps +``` + +### Windows commands + +```powershell +Set-Location "C:\NODEDC\dc-amd-connector" +docker compose ps +docker compose logs --tail 200 +docker inspect dc-amd-connector --format '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{end}} restart={{.HostConfig.RestartPolicy.Name}}' +``` + +### What `/status` means + +`/healthz` and `/status` currently return the same safe operational body. They +prove process/local state, not end-to-end provider reachability. + +Important fields: + +- `state`: `awaiting_pair` or `paired`; +- `forwarding`: disabled until paired, then `amd_connector_only`; +- `directEgress`: must always be `false`; +- `pool.activeSockets`, `idleSockets`, `queuedRequests`, `maxQueuedRequests`; +- `pool.byOrigin`: safe active/idle/queued counts per allowlisted origin; +- `metrics.requests`, `terminalRequests`, `inFlightRequests`; +- `completedResponses`, `failedRequests`, `clientAborts`, `streamFailures`; +- `openedTunnels`, `tunnelFailures`, `reusedSocketRequests`, `retries`; +- complete and partial byte counts; +- average/max queue, connection, TTFB, attempt, retry and total duration; +- per-host counters and the last short transport error. + +At all times: + +```text +requests = terminalRequests + inFlightRequests +``` + +At quiescence, `inFlightRequests` should return to zero. A violation indicates +a lifecycle/accounting defect. Counters are process-local and reset after a +container restart; pair state does not reset. + +Interpret latency by stage: + +| Symptom | Likely stage | +| --- | --- | +| High `queuedRequests` / `queueMs` | Per-origin eight-socket pool saturated or requests not terminating | +| High `connectionMs` / tunnel failures | NAS-to-AMD LAN, Windows connector, DNS, CONNECT or TLS setup | +| Low connection but high `ttfbMs` | VPN/provider latency or provider throttling | +| Low TTFB but high total duration | Slow/stalled response body or downstream backpressure | +| Rising `partialBytes` / `streamFailures` | Provider/VPN body died after headers | +| Rising `clientAborts` | Browser/navigation cancellation or caller timeout; not automatically an upstream failure | +| Low `reusedSocketRequests` with high `openedTunnels` | Keep-alive churn, route instability or origin fan-out | +| Retry spikes | Reused sockets are being reset; one bounded recovery is working but transport is unstable | + +JSON logs intentionally contain event names, approved hostname, short error +code and numeric timing only. They must not include a full URL, query string, +Ion bearer, connector credential or Map egress credential. Normal fast tile +requests are not logged individually. Useful events include: + +- `cesium_egress_tunnel_opened` +- `cesium_egress_reused_socket_retry` +- `cesium_egress_retry_completed` +- `cesium_egress_retry_failed` +- `cesium_egress_slow_or_failed` +- `cesium_egress_stream_failed` +- `cesium_egress_request_failed` +- `amd_connector_paired` + +A warm Map Gateway cache hit should not increase `dc-amd-proxy` request +counters. If it does, audit the cache-first ordering before tuning the proxy. + +## 7. Failure matrix + +| Observable result | Probable cause | Safe action | +| --- | --- | --- | +| NAS status unavailable | Container stopped/unhealthy or wrong NAS bind address | Inspect container/Compose state and logs; do not change NAS routes | +| `state=awaiting_pair` | No durable connector pair state | Verify intended AMD host, then run the one-time pairing tool | +| Pair HTTP 403 `pair_source_not_allowed` | Request did not originate from configured AMD LAN IP | Verify stable AMD IP, NAS host-mode listener and no proxy/NAT in the pair call | +| Pair HTTP 409 `already_paired` | Pair state already exists | Do not overwrite it; use the controlled migration/rotation procedure | +| `map_egress_unauthorized` | Map Gateway and NAS proxy do not share the runner-synced Map egress credential | Re-run canonical preflight/deploy investigation; never copy a token into `.env` by hand | +| `amd_connector_not_paired` | NAS runtime pair file absent/invalid | Pair from the allowed AMD source; inspect ownership only through canonical runner checks | +| Connector refused/timeout | Windows off, no user sign-in, Docker stopped, firewall/IP mismatch or connector unhealthy | Restore Windows/Docker/connector; warm cache should continue | +| `amd_connector_rejected` | Connector credential mismatch or CONNECT rejected | Stop; do not retry pairing blindly. Treat as pair-state/migration incident | +| `cesium_target_not_allowed` | Provider redirected/requested an unreviewed host or wrong scheme/port | Capture hostname safely, review necessity, update both allowlists through source | +| TLS timeout/certificate error | VPN path, DNS, time, CA or provider interception problem | Verify AMD route/VPN and NAS container time/CA; never disable TLS verification in production | +| `amd_upstream_timeout` | No provider headers within attempt timeout | Compare connection vs TTFB metrics; verify VPN/provider rather than increasing timeout first | +| Body/redirect idle timeout | Stream stopped making progress | Check VPN packet loss/provider; partial object must not enter Map cache | +| Provider HTTP 401/403 | Ion token, asset permission, referer restriction or provider account issue | Diagnose Map Gateway/provider configuration; transport is functioning if status is passed through | +| Provider HTTP 429 | Provider rate/throughput policy | Reduce cold fan-out and use cache; do not add unbounded sockets/retries | +| Rendering survives but cold areas fail | Expected cache-first degradation while AMD/VPN is down | Restore egress; preserve existing cache and avoid destructive cache resets | +| Queue grows and never drains | Saturation, hung lifecycle or insufficient abort propagation | Inspect in-flight/accounting invariant, body idle errors and client aborts before changing pool size | +| AMD host rebooted and does not recover | No Windows sign-in, Startup launcher missing, Docker not ready, container removed or VPN disconnected | Follow the explicit Windows reboot sequence | + +### Important VPN caveat + +The connector trusts the route Docker Desktop receives from Windows. It does +not cryptographically prove that a request used a particular VPN or country. +If Windows allows direct Internet egress when the VPN is off, the connector +may use that direct AMD-host route. It will still never fall back to direct NAS +egress, but that is a different guarantee. + +If VPN-only provider egress is mandatory, enforce it on the AMD host/VPN with +an outbound kill switch scoped to the connector workload/approved destinations +and test it with VPN on and off. The current installer creates only an inbound +NAS-only firewall rule; it does not install this outbound policy. + +## 8. Migration to another AMD machine + +Migration has two independent identities: stable LAN endpoint and connector +credential. A clean install on another machine creates a new credential, while +the current NAS pair endpoint is intentionally write-once. Changing only the +IP in Compose is therefore insufficient. + +### Required zero-guesswork migration sequence + +1. Keep the old connector active while preparing the replacement. +2. Assign the replacement a reserved `` and verify VPN/Docker + routing. +3. Install the reviewed connector package on the replacement using + `` and the same NAS address/connector port. +4. Verify local connector health and the safe CONNECT probe. +5. Create a reviewed source change that updates NAS connector host and allowed + pair source to ``. +6. Add a runner-supported, audited pair-rotation transition that can retire the + old NAS pair state and admit exactly one new pair. The transition must back + up state, never print either credential, fail closed and be idempotent. +7. Build, checksum, SMB-transfer, `plan`, and `apply` the exact versioned + artifact/runner change through the deploy canon. +8. Pair from the replacement machine, verify an end-to-end cold request and + confirm `directEgress=false` plus clean accounting. +9. Disable the old connector, then uninstall it only after acceptance. + +The current implementation does **not yet provide step 6 as a first-class +runner operation**. Until it exists, a fresh-secret migration is not fully +canonical. Do not work around this by deleting +`/volume1/docker/dc-amd-proxy/runtime/connector-access`, copying the credential +through chat/SMB, weakening `/api/pair`, or running two active pair sources. + +If an urgent migration is required before controlled rotation exists, open an +Ops change and implement/review that transition first. This limitation is +preferable to an undocumented secret reset. + +To remove a retired connector from Windows after the new path is accepted: + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass ` + -File "C:\NODEDC\dc-amd-connector\uninstall.ps1" ` + -Destination "C:\NODEDC\dc-amd-connector" +``` + +The uninstaller removes only this container, its NAS-only firewall rule, +Startup launcher and installation directory. It does not remove Docker Desktop +or unrelated containers. + +## 9. Deploy and rollback canon + +### Source/change acceptance before packaging + +Run from the service directory in a clean checkout: + +```bash +node --check server.mjs +npm run test:connection-pool +docker build --no-cache -t nodedc/dc-amd-proxy: . +git status --short +``` + +The smoke suite must cover pool reuse/saturation, aborts in each lifecycle +stage, exact terminal accounting, one safe retry, body idle timeout, redirect +credential boundaries and absence of secret markers in logs/status. + +Commit source and documentation together. Build an artifact from the commit, +not from an unknown dirty tree. The normal deployment sequence is: + +```text +source review -> tests -> commit -> artifact -> checksum -> SMB inbox +-> verify-install -> exact plan -> human plan review -> exact apply +-> container health -> proxy status -> end-to-end Map acceptance +``` + +Never put `.env`, `runtime`, tokens, logs, backup archives, Windows package +state or shell hooks inside a `dc-amd-proxy` app-overlay. + +### What the runner records + +Before applying, the runner creates: + +```text +/volume1/docker/nodedc-deploy/backups// +``` + +with manifest, file lists and `source-before.tgz`. Successful artifacts move +to `/volume1/docker/nodedc-deploy/applied`; failed artifacts move to +`/volume1/docker/nodedc-deploy/failed`. Applied and failed JSONL state lives +under `/volume1/docker/nodedc-deploy/state`. + +For this component the current runner does **not** perform generic automatic +source rollback after a failed post-copy build/health check. The backup is +evidence/recovery input, not permission for an improvised live extraction. + +### Canonical forward rollback + +Use a new patch ID and package the known-good service source from a separate +clean worktree. Do not reset or overwrite the active developer worktree: + +```bash +git worktree add "../NODEDC-rollback" +cd "../NODEDC-rollback" + +node infra/deploy-runner/build-dc-amd-proxy-artifact.mjs \ + dc-amd-proxy-rollback-- + +cd infra/deploy-artifacts +shasum -a 256 nodedc-dc-amd-proxy-rollback--.tgz \ + > nodedc-dc-amd-proxy-rollback--.tgz.sha256 +``` + +Transfer the exact pair to the inbox and run the same checksum, +`verify-install`, exact `plan`, review and exact `apply` sequence. The runtime +pair-state directory remains outside the source overlay and must be preserved. + +If the failed version changed a runtime contract, Compose contract or secret +format, stop and design an explicit rollback transition instead of assuming a +source-only forward rollback is safe. + +## 10. Capacity and product behavior + +The default eight active sockets are **per approved origin**, not global. A +Cesium session can use more than one origin, but every origin remains bounded. +Twenty users do not overwrite one another's transport state; they share the +pool and may queue. Cache objects are owned by Map Gateway, not by this proxy. + +The proxy has no tile index, no cache eviction and no “do not overwrite cache” +switch. Those belong to Map Gateway. It streams bytes and records transport +telemetry. Consequently: + +- proxy/VPN speed affects only cold/refresh acquisition; +- NAS cache read speed should be independent from AMD/VPN speed; +- increasing pool size cannot fix slow warm cache reads; +- cache fill concurrency and capacity policy must be tuned in Map Gateway; +- a browser reload should reuse NAS cache even if local browser cache is empty. + +## 11. Known limitations and evolution path + +The current path is production-usable but not yet a general infrastructure +egress product. Track these changes explicitly rather than accumulating +one-off live fixes: + +1. **Controlled pair rotation/migration.** Add runner-owned one-time rotation, + revocation and rollback state; optionally support overlapping old/new + credentials for a bounded cutover window. +2. **Remove the legacy `proxy-contur` secret source.** Generate/manage a + dedicated Map egress credential as first-class runner state. +3. **VPN-route enforcement.** Add a tested Windows outbound kill switch or a + dedicated egress appliance so VPN-off cannot become direct AMD egress. +4. **Unattended host recovery.** Replace user-sign-in-dependent Docker Desktop + with a managed Windows service, signed installer or dedicated Linux egress + node if 24/7 cold-cache availability is required. +5. **Pair-channel hardening.** Replace source-IP-only HTTP pairing with a + short-lived nonce plus authenticated/encrypted handoff or mutual TLS. +6. **Two-level health.** Keep local `/healthz`, add a rate-limited synthetic + egress readiness check that proves DNS/CONNECT/TLS without consuming or + exposing the master Ion credential. +7. **Durable metrics.** Export the safe counters to the platform metrics stack; + process-local `/status` currently resets on restart. +8. **Central allowlist contract.** Generate identical proxy/connector host + policies from one reviewed provider contract and test that they cannot + drift. +9. **Circuit breaking and backoff.** Add per-origin fail-fast state for a dead + AMD/VPN path so many cold misses do not occupy all queues until timeout. +10. **High availability.** Define an explicit active/standby connector model, + health-based selection and credential isolation before adding a second + workstation. +11. **Signed Windows distribution/update.** Provide a versioned package builder, + signature verification and in-place upgrade/rollback rather than manual + archive handling. +12. **SLOs and load tests.** Establish cache-hit, cold-TTFB, queue-depth, + abort-rate and recovery-time objectives using realistic concurrent Map + Page sessions. + +Until those changes land, preserve the narrow boundary: official allowlisted +HTTPS only, no direct NAS fallback, no global workstation proxy, no secret in +source, and every NAS mutation through a reviewed data-only artifact plus the +canonical runner. diff --git a/services/dc-amd-proxy/README.md b/services/dc-amd-proxy/README.md new file mode 100644 index 0000000..5a586e6 --- /dev/null +++ b/services/dc-amd-proxy/README.md @@ -0,0 +1,84 @@ +# DC AMD Proxy + +Полный воспроизводимый install/recovery/migration-контракт находится в +[`OPERATIONS.md`](./OPERATIONS.md). Этот README фиксирует краткую runtime-модель; +операционные изменения должны одновременно обновлять runbook. + +`dc-amd-proxy` is the dedicated, controllable boundary between NODE.DC Map +Gateway and the adjacent AMD VPN machine. It is deliberately independent from +the historic `proxy-contur` service. + +After pairing it has no direct Cesium egress: every approved Cesium/Bing +request is tunnelled through the authenticated AMD connector +(`172.22.0.183:8791`). The NAS always addresses this stable LAN endpoint; +the AMD host alone owns its changing VPN exit. + +This is a single restricted transport path, not a per-tile relay chain. The +NAS keeps a small, bounded pool of HTTP/1.1 TLS tunnels per approved upstream +host, so adjacent tile requests reuse the existing `NAS → AMD → VPN` channel. +The pool is limited to eight active and four idle sockets per host by default; +it cannot become a general-purpose proxy or consume an unbounded amount of the +workstation's network resources. + +`/status` exposes only safe operational counters: active/idle/queued sockets +per approved origin, the retained peak queue, logical/in-flight/terminal +requests, tunnels opened, reused sockets, bounded retries, complete and partial +response bytes, separate queue/CONNECT+TLS/TTFB/retry/total timing, body-stream +failures, client aborts, and the last short error code. Every accepted egress +request is accounted exactly once even when the browser disconnects while it +is queued, connecting, following a redirect, waiting for headers, or retrying. +The status never contains an Ion token, connector token, full URL, or query +parameters. + +An upstream `Authorization` header is retained only across same-origin +redirects. A redirect from the Cesium API origin to an assets or Bing origin +is followed only after that header is removed, so an API/master bearer cannot +cross provider credential boundaries. + +Only an idempotent `GET`/`HEAD` that loses an already reused keep-alive socket +is retried, once. The failed reused-socket attempt and the terminal retry +attempt are logged separately with numeric transport timings and no URL or +credential fields. A response body that makes no progress for 30 seconds is +terminated and counted as a stream failure, so a stalled VPN connection cannot +occupy one of the eight per-origin sockets forever. These limits are local to +Cesium egress and do not modify routing or connectivity of the AMD workstation. + +## Pairing and operational state + +The active service has an explicitly narrow first-pair contract: + +- `POST /api/pair` accepts exactly one connector access token and only from + the configured AMD LAN source address; +- the AMD operator invokes a supplied local script; it reads the token inside + the AMD installation and sends it without printing it or putting it in chat; +- the NAS writes the paired value only to its private runtime directory, never + to `.env`, SMB inbox, a deploy artifact, browser code, or logs; +- the Map Gateway fetch contract (`/proxy/cesium/fetch`) remains private to + the NAS LAN address and still requires the runner-synchronised map egress + token; +- the service follows only the fixed Cesium Ion/Bing host allowlist and cannot + fall back to direct NAS Internet egress. + +## Deployment order + +1. Deploy this service and confirm `/healthz` reports `awaiting_pair`. + Map Gateway remains on its existing egress during this step. +2. On AMD, run the supplied one-time pairing script. It sends the locally-held + connector secret directly to the NAS and reports no secret value. +3. Verify the private Map Gateway request through this service. +4. Apply the separate, minimal Map Gateway switch patch. It changes only its + private egress hostname from the historical service to `dc-amd-proxy`. + +The initial 001 deployment was intentionally inert. It is superseded by this +active configuration; it did not alter NAS routing, the NAS VPN/Tailscale +configuration, or the Windows workstation's VPN configuration. + +## Why host networking is narrow here + +Synology's Docker published-port proxy replaces the caller address before it +reaches a bridged container. That makes a strict one-time AMD source-address +check impossible. This service therefore uses the host network namespace but +binds its own HTTP listener only to `172.22.0.222:8790`; it is not bound to all +NAS interfaces. No host route, VPN, DNS, Tailscale setting or firewall rule is +changed. The forwarding code still has one hard-coded outbound path only: +the authenticated AMD connector. diff --git a/services/dc-amd-proxy/docker-compose.yml b/services/dc-amd-proxy/docker-compose.yml new file mode 100644 index 0000000..ff1c720 --- /dev/null +++ b/services/dc-amd-proxy/docker-compose.yml @@ -0,0 +1,47 @@ +services: + dc-amd-proxy: + build: + context: . + dockerfile: Dockerfile + container_name: dc-amd-proxy + image: nodedc/dc-amd-proxy:local + restart: unless-stopped + # Preserve the real AMD source address for the one-time pairing request. + # The process itself binds only to the NAS LAN address below; this does not + # change NAS routes, VPN, DNS, Tailscale, or any other host service. + network_mode: host + environment: + PORT: "8790" + DC_AMD_PROXY_BIND_ADDRESS: ${DC_AMD_PROXY_BIND_ADDRESS:-172.22.0.222} + DC_AMD_CONNECTOR_HOST: ${DC_AMD_CONNECTOR_HOST:-172.22.0.183} + DC_AMD_CONNECTOR_PORT: ${DC_AMD_CONNECTOR_PORT:-8791} + DC_AMD_PAIR_ALLOWED_SOURCE: ${DC_AMD_PAIR_ALLOWED_SOURCE:-172.22.0.183} + DC_AMD_CONNECTOR_TOKEN_FILE: /var/lib/dc-amd-proxy/connector-access + DC_AMD_MAP_EGRESS_TOKEN_FILE: /run/nodedc-secrets/map-egress-proxy-token + DC_AMD_BODY_IDLE_TIMEOUT_SECONDS: ${DC_AMD_BODY_IDLE_TIMEOUT_SECONDS:-30} + volumes: + - type: bind + source: /volume1/docker/nodedc-platform/secrets/map-egress-proxy-token + target: /run/nodedc-secrets/map-egress-proxy-token + read_only: true + - type: bind + source: ./runtime + target: /var/lib/dc-amd-proxy + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=8m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD-SHELL + - > + node -e "fetch('http://'+process.env.DC_AMD_PROXY_BIND_ADDRESS+':8790/healthz') + .then(r => process.exit(r.ok ? 0 : 1)) + .catch(() => process.exit(1))" + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s diff --git a/services/dc-amd-proxy/package.json b/services/dc-amd-proxy/package.json new file mode 100644 index 0000000..cb6323c --- /dev/null +++ b/services/dc-amd-proxy/package.json @@ -0,0 +1,10 @@ +{ + "name": "dc-amd-proxy", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start": "node server.mjs", + "test:connection-pool": "node scripts/smoke-connection-pool.mjs" + } +} diff --git a/services/dc-amd-proxy/scripts/smoke-connection-pool.mjs b/services/dc-amd-proxy/scripts/smoke-connection-pool.mjs new file mode 100644 index 0000000..54cdb33 --- /dev/null +++ b/services/dc-amd-proxy/scripts/smoke-connection-pool.mjs @@ -0,0 +1,499 @@ +import assert from "node:assert/strict"; +import { execFile as execFileCallback, spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer as createHttpServer, request as createHttpRequest } from "node:http"; +import { createServer as createHttpsServer } from "node:https"; +import { connect, createServer as createNetServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const execFile = promisify(execFileCallback); +const root = await mkdtemp(join(tmpdir(), "nodedc-amd-pool-smoke-")); +const upstreamPort = await freePort(); +const connectorPort = await freePort(); +const proxyPort = await freePort(); +const upstreamSockets = new Set(); +const connectorSockets = new Set(); +let upstream; +let connector; +let proxy; +let proxyOutput = ""; +let tunnelCount = 0; +let upstreamRequests = 0; +let retryAttempts = 0; +let nonReusedResetAttempts = 0; +let retryLimitAttempts = 0; +let retryAbortAttempts = 0; +let queuedAbortAttempts = 0; +let nextConnectorDelayMs = 15; +const redirectCredentials = {}; +let holdReleased = false; +let releaseHold; +let markHoldStarted; +let markPreHeaderSeen; +let markPreHeaderClosed; +let markIdleClosed; +let markDelayedConnectorSeen; +let markDelayedConnectorClosed; +let markRedirectSeen; +let markRedirectClosed; +let markRetryAbortSecondSeen; +let markRetryAbortSecondClosed; +const holdRelease = new Promise((resolve) => { releaseHold = resolve; }); +const holdStarted = new Promise((resolve) => { markHoldStarted = resolve; }); +const preHeaderSeen = new Promise((resolve) => { markPreHeaderSeen = resolve; }); +const preHeaderClosed = new Promise((resolve) => { markPreHeaderClosed = resolve; }); +const idleClosed = new Promise((resolve) => { markIdleClosed = resolve; }); +const delayedConnectorSeen = new Promise((resolve) => { markDelayedConnectorSeen = resolve; }); +const delayedConnectorClosed = new Promise((resolve) => { markDelayedConnectorClosed = resolve; }); +const redirectSeen = new Promise((resolve) => { markRedirectSeen = resolve; }); +const redirectClosed = new Promise((resolve) => { markRedirectClosed = resolve; }); +const retryAbortSecondSeen = new Promise((resolve) => { markRetryAbortSecondSeen = resolve; }); +const retryAbortSecondClosed = new Promise((resolve) => { markRetryAbortSecondClosed = resolve; }); + +try { + const keyPath = join(root, "key.pem"); + const certificatePath = join(root, "certificate.pem"); + await execFile("openssl", [ + "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", keyPath, + "-out", certificatePath, + "-subj", "/CN=api.cesium.com", + "-days", "1", + ]); + + upstream = createHttpsServer({ key: await readFile(keyPath), cert: await readFile(certificatePath) }, (request, response) => { + upstreamRequests += 1; + if (request.url === "/retry") { + retryAttempts += 1; + if (retryAttempts === 1) { + setTimeout(() => request.socket.destroy(), 25); + return; + } + response.writeHead(200, { "content-type": "text/plain" }); + response.end("retry-ok"); + return; + } + if (request.url?.startsWith("/hold/")) { + markHoldStarted(); + const finish = () => { + response.writeHead(200, { "content-type": "text/plain" }); + response.end(`hold-${request.url.slice("/hold/".length)}`); + }; + if (holdReleased) finish(); + else void holdRelease.then(finish); + return; + } + if (request.url === "/preheader") { + markPreHeaderSeen(); + request.socket.once("close", markPreHeaderClosed); + return; + } + if (request.url === "/redirect-hold") { + markRedirectSeen(); + request.socket.once("close", markRedirectClosed); + response.writeHead(302, { location: "https://api.cesium.com/redirect-target", "content-type": "text/plain" }); + response.write("redirect-partial"); + return; + } + if (request.url === "/redirect-same-origin") { + redirectCredentials.sameSource = request.headers.authorization; + response.writeHead(302, { location: "/same-origin-target" }); + response.end("same-origin-redirect"); + return; + } + if (request.url === "/same-origin-target") { + redirectCredentials.sameTarget = request.headers.authorization; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("same-origin-ok"); + return; + } + if (request.url === "/redirect-assets-origin") { + redirectCredentials.assetsSource = request.headers.authorization; + response.writeHead(302, { location: "https://assets.ion.cesium.com/assets-origin-target" }); + response.end("assets-origin-redirect"); + return; + } + if (request.url === "/assets-origin-target") { + redirectCredentials.assetsHost = request.headers.host; + redirectCredentials.assetsTarget = request.headers.authorization; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("assets-origin-ok"); + return; + } + if (request.url === "/redirect-bing-origin") { + redirectCredentials.bingSource = request.headers.authorization; + response.writeHead(302, { location: "https://dev.virtualearth.net/bing-origin-target" }); + response.end("bing-origin-redirect"); + return; + } + if (request.url === "/bing-origin-target") { + redirectCredentials.bingHost = request.headers.host; + redirectCredentials.bingTarget = request.headers.authorization; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("bing-origin-ok"); + return; + } + if (request.url === "/non-reused-reset") { + nonReusedResetAttempts += 1; + setTimeout(() => request.socket.destroy(), 25); + return; + } + if (request.url === "/queued-abort") { + queuedAbortAttempts += 1; + response.writeHead(200, { "content-type": "text/plain" }); + response.end("queue-abort-reached-upstream"); + return; + } + if (request.url === "/retry-limit") { + retryLimitAttempts += 1; + setTimeout(() => request.socket.destroy(), 25); + return; + } + if (request.url === "/retry-abort") { + retryAbortAttempts += 1; + if (retryAbortAttempts === 1) { + setTimeout(() => request.socket.destroy(), 25); + return; + } + markRetryAbortSecondSeen(); + request.socket.once("close", markRetryAbortSecondClosed); + return; + } + if (request.url === "/idle") { + request.socket.once("close", markIdleClosed); + response.writeHead(200, { "content-type": "text/plain" }); + response.write("partial"); + return; + } + const responseNumber = upstreamRequests; + const finish = () => { + response.writeHead(200, { "content-type": "text/plain" }); + response.end(`tile-${responseNumber}`); + }; + if (request.url?.startsWith("/tile/0?")) setTimeout(finish, 25); + else finish(); + }); + upstream.on("connection", (socket) => { + upstreamSockets.add(socket); + socket.once("close", () => upstreamSockets.delete(socket)); + }); + upstream.listen(upstreamPort, "127.0.0.1"); + await once(upstream, "listening"); + + connector = createHttpServer((_request, response) => { + response.writeHead(405); + response.end(); + }); + connector.on("connection", (socket) => { + connectorSockets.add(socket); + socket.once("close", () => connectorSockets.delete(socket)); + }); + connector.on("connect", (_request, clientSocket, head) => { + tunnelCount += 1; + // The fake connector deliberately ignores the requested hostname and + // connects to a local TLS fixture. Production still enforces the strict + // Cesium/Bing allowlist before opening this tunnel. + const connectorDelayMs = nextConnectorDelayMs; + nextConnectorDelayMs = 15; + if (connectorDelayMs > 100) { + markDelayedConnectorSeen(); + clientSocket.once("close", markDelayedConnectorClosed); + } + const target = connect(upstreamPort, "127.0.0.1"); + target.once("connect", () => { + setTimeout(() => { + if (clientSocket.destroyed || target.destroyed) return; + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length) target.write(head); + clientSocket.pipe(target); + target.pipe(clientSocket); + }, connectorDelayMs); + }); + target.once("error", () => clientSocket.destroy()); + clientSocket.once("error", () => target.destroy()); + clientSocket.once("close", () => target.destroy()); + }); + connector.listen(connectorPort, "127.0.0.1"); + await once(connector, "listening"); + + const connectorToken = `connector-secret-${"a".repeat(48)}`; + const mapToken = "map-pool-smoke-secret"; + const providerToken = "provider-secret-marker"; + const connectorTokenFile = join(root, "connector-token"); + const mapTokenFile = join(root, "map-token"); + await writeFile(connectorTokenFile, `${connectorToken}\n`, { mode: 0o600 }); + await writeFile(mapTokenFile, `${mapToken}\n`, { mode: 0o600 }); + + proxy = spawn(process.execPath, ["server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + NODE_ENV: "test", + PORT: String(proxyPort), + DC_AMD_PROXY_BIND_ADDRESS: "127.0.0.1", + DC_AMD_CONNECTOR_HOST: "127.0.0.1", + DC_AMD_CONNECTOR_PORT: String(connectorPort), + DC_AMD_PAIR_ALLOWED_SOURCE: "127.0.0.1", + DC_AMD_CONNECTOR_TOKEN_FILE: connectorTokenFile, + DC_AMD_MAP_EGRESS_TOKEN_FILE: mapTokenFile, + DC_AMD_PROXY_TEST_ALLOW_INSECURE_TLS: "true", + DC_AMD_POOL_MAX_SOCKETS: "1", + DC_AMD_POOL_MAX_FREE_SOCKETS: "1", + DC_AMD_SLOW_REQUEST_SECONDS: "30", + DC_AMD_UPSTREAM_TIMEOUT_SECONDS: "2", + DC_AMD_BODY_IDLE_TIMEOUT_SECONDS: "1", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + proxy.stdout.on("data", (chunk) => { proxyOutput += String(chunk); }); + proxy.stderr.on("data", (chunk) => { proxyOutput += String(chunk); }); + await waitForProxy(proxyPort, proxy, () => proxyOutput); + + for (let index = 0; index < 4; index += 1) { + const target = `https://api.cesium.com/tile/${index}${index === 0 ? `?access_token=${providerToken}` : ""}`; + const response = await proxyFetch(proxyPort, mapToken, target); + assert.equal(response.status, 200); + assert.equal(await response.text(), `tile-${index + 1}`); + } + assert.equal(upstreamRequests, 4); + assert.equal(tunnelCount, 1); + + const retryResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/retry"); + assert.equal(retryResponse.status, 200); + assert.equal(await retryResponse.text(), "retry-ok"); + assert.equal(retryAttempts, 2); + + const redirectAuthorization = "Bearer master-api-secret-marker"; + const redirectHeaders = { "x-nodedc-cesium-authorization": redirectAuthorization }; + const sameOriginResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/redirect-same-origin", undefined, redirectHeaders); + assert.equal(sameOriginResponse.status, 200); + assert.equal(await sameOriginResponse.text(), "same-origin-ok"); + const assetsOriginResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/redirect-assets-origin", undefined, redirectHeaders); + assert.equal(assetsOriginResponse.status, 200); + assert.equal(await assetsOriginResponse.text(), "assets-origin-ok"); + const bingOriginResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/redirect-bing-origin", undefined, redirectHeaders); + assert.equal(bingOriginResponse.status, 200); + assert.equal(await bingOriginResponse.text(), "bing-origin-ok"); + assert.equal(redirectCredentials.sameSource, redirectAuthorization); + assert.equal(redirectCredentials.sameTarget, redirectAuthorization); + assert.equal(redirectCredentials.assetsSource, redirectAuthorization); + assert.equal(redirectCredentials.assetsTarget, undefined); + assert.equal(redirectCredentials.assetsHost, "assets.ion.cesium.com"); + assert.equal(redirectCredentials.bingSource, redirectAuthorization); + assert.equal(redirectCredentials.bingTarget, undefined); + assert.equal(redirectCredentials.bingHost, "dev.virtualearth.net"); + + const held = Array.from({ length: 4 }, (_, index) => ( + rawProxyRequest(proxyPort, mapToken, `https://api.cesium.com/hold/${index}`).promise + )); + await withTimeout(holdStarted, 1_000, "first saturated request did not reach upstream"); + const queuedAbortRequest = rawProxyRequest(proxyPort, mapToken, "https://api.cesium.com/queued-abort"); + const saturatedStatus = await waitFor(async () => { + const status = await (await fetch(`http://127.0.0.1:${proxyPort}/status`)).json(); + return status.transport.pool.queuedRequests >= 4 ? status : null; + }, 1_000, "agent queue never reached saturation"); + assert.equal(saturatedStatus.transport.pool.maxQueuedRequests >= 4, true); + queuedAbortRequest.abort(); + await assert.rejects(withTimeout(queuedAbortRequest.promise, 1_000, "queued abort did not reject client request")); + await delay(100); + holdReleased = true; + releaseHold(); + const heldResults = await withTimeout(Promise.all(held), 2_000, "saturated requests did not drain"); + for (const [index, result] of heldResults.entries()) { + assert.equal(result.status, 200); + assert.equal(result.body, `hold-${index}`); + } + assert.equal(queuedAbortAttempts, 0); + + const redirectAbortRequest = rawProxyRequest(proxyPort, mapToken, "https://api.cesium.com/redirect-hold"); + await withTimeout(redirectSeen, 1_000, "redirect request did not reach upstream"); + redirectAbortRequest.abort(); + await assert.rejects(withTimeout(redirectAbortRequest.promise, 1_000, "redirect abort did not reject client request")); + await withTimeout(redirectClosed, 1_000, "redirect abort did not close redirect body socket"); + + const preHeaderRequest = rawProxyRequest(proxyPort, mapToken, "https://api.cesium.com/preheader"); + await withTimeout(preHeaderSeen, 1_000, "pre-header request did not reach upstream"); + preHeaderRequest.abort(); + await assert.rejects(withTimeout(preHeaderRequest.promise, 1_000, "pre-header abort did not reject client request")); + await withTimeout(preHeaderClosed, 1_000, "pre-header abort did not close upstream socket"); + + // With no pooled socket left, delay the next CONNECT response and abort the + // browser-side request while that tunnel is still being established. + nextConnectorDelayMs = 500; + const connectAbortRequest = rawProxyRequest(proxyPort, mapToken, "https://api.cesium.com/connect-abort"); + await withTimeout(delayedConnectorSeen, 1_000, "CONNECT-abort request did not reach connector"); + connectAbortRequest.abort(); + await assert.rejects(withTimeout(connectAbortRequest.promise, 1_000, "CONNECT abort did not reject client request")); + await withTimeout(delayedConnectorClosed, 1_000, "CONNECT abort did not close connector socket"); + await waitFor(async () => { + const status = await (await fetch(`http://127.0.0.1:${proxyPort}/status`)).json(); + return status.transport.metrics.inFlightRequests === 0 ? status : null; + }, 1_000, "CONNECT abort was not terminally accounted"); + + // The abort above removed the pooled socket. A reset on the next freshly + // opened socket must fail directly, never trigger the safe reused-socket retry. + const nonReusedResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/non-reused-reset"); + assert.equal(nonReusedResponse.status, 502); + assert.equal(nonReusedResetAttempts, 1); + + const warmResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/warm"); + assert.equal(warmResponse.status, 200); + await warmResponse.text(); + + const retryAbortRequest = rawProxyRequest(proxyPort, mapToken, "https://api.cesium.com/retry-abort"); + await withTimeout(retryAbortSecondSeen, 1_000, "retry abort did not reach its second attempt"); + retryAbortRequest.abort(); + await assert.rejects(withTimeout(retryAbortRequest.promise, 1_000, "retry abort did not reject client request")); + await withTimeout(retryAbortSecondClosed, 1_000, "retry abort did not close second-attempt socket"); + assert.equal(retryAbortAttempts, 2); + + const retryLimitWarmResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/warm-after-retry-abort"); + assert.equal(retryLimitWarmResponse.status, 200); + await retryLimitWarmResponse.text(); + + // First failure is on a reused socket and is retried once. The second + // failure is terminal: retry remains strictly bounded to two attempts. + const retryLimitResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/retry-limit"); + assert.equal(retryLimitResponse.status, 502); + assert.equal(retryLimitAttempts, 2); + + const idleResponse = await proxyFetch(proxyPort, mapToken, "https://api.cesium.com/idle"); + assert.equal(idleResponse.status, 200); + await assert.rejects(withTimeout(idleResponse.text(), 2_000, "idle body timeout did not terminate downstream")); + await withTimeout(idleClosed, 1_000, "idle timeout did not close upstream socket"); + + const status = await (await fetch(`http://127.0.0.1:${proxyPort}/status`)).json(); + const metrics = status.transport.metrics; + assert.equal(metrics.requests, 22); + assert.equal(metrics.terminalRequests, 22); + assert.equal(metrics.inFlightRequests, 0); + assert.equal(metrics.completedResponses, 14); + assert.equal(metrics.failedRequests, 3); + assert.equal(metrics.retries, 3); + assert.equal(metrics.streamFailures, 1); + assert.equal(metrics.clientAborts, 5); + assert.equal(metrics.responseBytes > 0, true); + assert.equal(metrics.partialBytes >= Buffer.byteLength("partial"), true); + assert.equal(metrics.reusedSocketRequests >= 8, true); + assert.equal(metrics.timing.queueMs.max >= 100, true); + assert.equal(metrics.timing.connectionMs.max > 0, true); + assert.equal(metrics.timing.ttfbMs.max >= 20, true); + assert.equal(metrics.timing.retryMs.max >= 25, true); + assert.equal(status.transport.pool.queuedRequests, 0); + assert.equal(status.transport.pool.maxQueuedRequests >= 4, true); + assert.equal(status.transport.pool.maxSockets, 1); + assert.equal(status.transport.metrics.byHost["api.cesium.com"].terminalRequests, 22); + assert.equal(proxyOutput.includes('"event":"cesium_egress_reused_socket_retry"'), true); + assert.equal(proxyOutput.includes('"event":"cesium_egress_retry_completed"'), true); + assert.equal(proxyOutput.includes('"event":"cesium_egress_retry_failed"'), true); + for (const secret of [connectorToken, mapToken, providerToken, redirectAuthorization]) assert.equal(proxyOutput.includes(secret), false); + console.log("ok: aborts propagate, queue peaks are retained, retries are bounded, timings are separated, and terminal accounting is exact"); +} catch (error) { + if (proxyOutput) console.error(proxyOutput); + throw error; +} finally { + if (proxy && proxy.exitCode === null && proxy.signalCode === null) { + proxy.kill("SIGTERM"); + try { + await withTimeout(once(proxy, "exit"), 2_000, "proxy did not stop after SIGTERM"); + } catch { + proxy.kill("SIGKILL"); + await withTimeout(once(proxy, "exit"), 2_000, "proxy did not stop after SIGKILL").catch(() => undefined); + } + } + await closeServer(connector, connectorSockets); + await closeServer(upstream, upstreamSockets); + await rm(root, { recursive: true, force: true }); +} + +function proxyFetch(port, token, target, signal, extraHeaders = {}) { + return fetch(`http://127.0.0.1:${port}/proxy/cesium/fetch?url=${encodeURIComponent(target)}`, { + headers: { "x-proxy-token": token, ...extraHeaders }, + signal, + }); +} + +function rawProxyRequest(port, token, target) { + let request; + const promise = new Promise((resolve, reject) => { + request = createHttpRequest({ + host: "127.0.0.1", + port, + method: "GET", + path: `/proxy/cesium/fetch?url=${encodeURIComponent(target)}`, + headers: { "x-proxy-token": token }, + agent: false, + }, (response) => { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.once("end", () => resolve({ status: response.statusCode, body: Buffer.concat(chunks).toString("utf8") })); + response.once("error", reject); + }); + request.once("error", reject); + request.end(); + }); + // Register a rejection observer immediately; the test asserts the same + // original promise after it deliberately destroys the request. + void promise.catch(() => undefined); + return { + promise, + abort() { request.destroy(new Error("fixture_client_abort")); }, + }; +} + +async function freePort() { + const server = createNetServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address === "object"); + const closed = once(server, "close"); + server.close(); + await closed; + return address.port; +} + +async function waitForProxy(port, child, output) { + for (let attempt = 0; attempt < 80; attempt += 1) { + if (child.exitCode !== null) throw new Error(`proxy_exited:${child.exitCode}:${output()}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`); + if (response.ok) return; + } catch { /* service is still starting */ } + await delay(50); + } + throw new Error(`proxy_start_timeout:${output()}`); +} + +async function waitFor(read, timeoutMs, message) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const value = await read(); + if (value) return value; + await delay(20); + } + throw new Error(message); +} + +function withTimeout(promise, timeoutMs, message) { + let timeout; + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + return Promise.race([promise, deadline]).finally(() => clearTimeout(timeout)); +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function closeServer(server, sockets) { + for (const socket of sockets) socket.destroy(); + if (!server?.listening) return; + const closed = once(server, "close"); + server.close(); + await withTimeout(closed, 2_000, "fixture server did not close").catch(() => undefined); +} diff --git a/services/dc-amd-proxy/server.mjs b/services/dc-amd-proxy/server.mjs new file mode 100644 index 0000000..a6c0999 --- /dev/null +++ b/services/dc-amd-proxy/server.mjs @@ -0,0 +1,987 @@ +import { timingSafeEqual } from "node:crypto"; +import { link, mkdir, open, readFile, unlink } from "node:fs/promises"; +import { createServer } from "node:http"; +import https from "node:https"; +import { connect as connectNet } from "node:net"; +import { dirname } from "node:path"; +import { connect as connectTls } from "node:tls"; + +const allowedHosts = new Set([ + "api.cesium.com", + "assets.ion.cesium.com", + "dev.virtualearth.net", + "ecn.t0.tiles.virtualearth.net", + "ecn.t1.tiles.virtualearth.net", + "ecn.t2.tiles.virtualearth.net", + "ecn.t3.tiles.virtualearth.net", +]); +const connectionTimingSymbol = Symbol("nodedc.connection-timing"); +const config = await readConfig(); +const transport = createTransport(config); + +const server = createServer(async (request, response) => { + const url = new URL(request.url || "/", "http://localhost"); + try { + if (request.method === "GET" && ["/healthz", "/status"].includes(url.pathname)) return writeJson(response, 200, await statusBody()); + if (request.method === "POST" && url.pathname === "/api/pair") return await pairConnector(request, response); + if (["GET", "HEAD"].includes(request.method || "") && url.pathname === "/proxy/cesium/fetch") return await forwardCesiumRequest(request, response, url); + return writeJson(response, 404, { ok: false, error: "not_found" }); + } catch (error) { + if (response.destroyed || response.headersSent || response.writableEnded) { + if (!response.writableEnded) response.destroy(); + return; + } + const errorCode = safeError(error); + const status = Number(error?.statusCode || 502); + console.warn(JSON.stringify({ event: "cesium_egress_request_failed", error: errorCode, status, ...telemetryLogFields(error?.telemetry) })); + return writeJson(response, status, { ok: false, error: errorCode }, { "x-nodedc-egress-error": errorCode }); + } +}); + +server.listen(config.port, config.bindAddress, () => console.log(JSON.stringify({ event: "dc_amd_proxy_started", bindAddress: config.bindAddress, port: config.port, connector: `${config.connectorHost}:${config.connectorPort}`, directEgress: false }))); +for (const signal of ["SIGINT", "SIGTERM"]) process.on(signal, () => server.close(() => transport.close(() => process.exit(0)))); + +async function readConfig() { + const nodeEnv = String(process.env.NODE_ENV || "production").trim(); + return { + port: parsePort(process.env.PORT, 8790), + bindAddress: parseIpv4(process.env.DC_AMD_PROXY_BIND_ADDRESS || "172.22.0.222", "bind_address"), + connectorHost: parseIpv4(process.env.DC_AMD_CONNECTOR_HOST || "172.22.0.183", "connector_host"), + connectorPort: parsePort(process.env.DC_AMD_CONNECTOR_PORT, 8791), + pairAllowedSource: parseIpv4(process.env.DC_AMD_PAIR_ALLOWED_SOURCE || "172.22.0.183", "pair_allowed_source"), + connectorTokenFile: String(process.env.DC_AMD_CONNECTOR_TOKEN_FILE || "/var/lib/dc-amd-proxy/connector-access").trim(), + mapToken: Buffer.from(await readSecretFile(process.env.DC_AMD_MAP_EGRESS_TOKEN_FILE || "/run/nodedc-secrets/map-egress-proxy-token", "map_egress_token"), "utf8"), + connectTimeoutMs: parseDuration(process.env.DC_AMD_CONNECT_TIMEOUT_SECONDS, 20), + upstreamTimeoutMs: parseDuration(process.env.DC_AMD_UPSTREAM_TIMEOUT_SECONDS, 30), + bodyIdleTimeoutMs: parseDuration(process.env.DC_AMD_BODY_IDLE_TIMEOUT_SECONDS, 30), + poolMaxSockets: parsePoolSize(process.env.DC_AMD_POOL_MAX_SOCKETS, 8), + poolMaxFreeSockets: parsePoolSize(process.env.DC_AMD_POOL_MAX_FREE_SOCKETS, 4), + slowRequestMs: parseDuration(process.env.DC_AMD_SLOW_REQUEST_SECONDS, 2), + allowInsecureTls: nodeEnv === "test" && parseBoolean(process.env.DC_AMD_PROXY_TEST_ALLOW_INSECURE_TLS, false), + }; +} + +async function statusBody() { + const paired = Boolean(await readConnectorToken()); + return { + ok: true, + service: "dc-amd-proxy", + state: paired ? "paired" : "awaiting_pair", + forwarding: paired ? "amd_connector_only" : "disabled", + directEgress: false, + connector: `${config.connectorHost}:${config.connectorPort}`, + transport: transport.status(), + }; +} + +async function pairConnector(request, response) { + if (normalizeAddress(request.socket.remoteAddress) !== config.pairAllowedSource) return writeJson(response, 403, { ok: false, error: "pair_source_not_allowed" }); + if (await readConnectorToken()) return writeJson(response, 409, { ok: false, error: "already_paired" }); + const body = await readJsonBody(request, 1024); + const token = String(body?.connectorAccessToken || "").trim(); + if (!isSecret(token)) return writeJson(response, 400, { ok: false, error: "connector_access_token_invalid" }); + await persistConnectorToken(token); + console.log(JSON.stringify({ event: "amd_connector_paired" })); + return writeJson(response, 201, { ok: true, state: "paired" }); +} + +async function forwardCesiumRequest(request, response, requestUrl) { + if (!isMapAuthorized(request.headers["x-proxy-token"])) return writeJson(response, 401, { ok: false, error: "map_egress_unauthorized" }); + const target = parseTarget(requestUrl.searchParams.get("url") || ""); + const startedAt = Date.now(); + const terminal = transport.beginRequest(target.hostname); + const controller = new AbortController(); + let telemetry = emptyTelemetry(); + let upstream; + let responseBytes = 0; + let upstreamEnded = false; + let downstreamFinished = false; + let clientAborted = false; + let terminalFinished = false; + let bodyIdleTimer; + let abortUpstream; + const clearBodyIdleTimer = () => { + if (bodyIdleTimer) clearTimeout(bodyIdleTimer); + bodyIdleTimer = undefined; + }; + const cleanup = () => { + clearBodyIdleTimer(); + request.off("aborted", onClientAbort); + response.off("close", onClientAbort); + if (abortUpstream) controller.signal.removeEventListener("abort", abortUpstream); + }; + const finish = (outcome) => { + if (terminalFinished) return false; + terminalFinished = terminal.complete({ + ...outcome, + durationMs: Date.now() - startedAt, + bytes: responseBytes, + telemetry, + }); + if (terminalFinished) cleanup(); + return terminalFinished; + }; + function onClientAbort() { + if (downstreamFinished || terminalFinished) return; + clientAborted = true; + if (!controller.signal.aborted) controller.abort(abortError()); + // Before headers, requestOnce owns the active/queued ClientRequest and its + // signal. The await/catch below records its complete attempt telemetry. + if (upstream) finish({ type: "client-abort" }); + } + const armBodyIdleTimer = () => { + clearBodyIdleTimer(); + bodyIdleTimer = setTimeout(() => { + const error = typedError("amd_upstream_body_idle_timeout", 502); + error.code = "ETIMEDOUT"; + upstream.destroy(error); + }, config.bodyIdleTimeoutMs); + bodyIdleTimer.unref?.(); + }; + request.once("aborted", onClientAbort); + response.once("close", onClientAbort); + + try { + const connectorToken = await readConnectorToken(); + if (clientAborted || controller.signal.aborted) { + finish({ type: "client-abort" }); + return; + } + if (!connectorToken) { + finish({ type: "request-failure", errorCode: "amd_connector_not_paired" }); + return writeJson(response, 503, { ok: false, error: "amd_connector_not_paired" }, { "x-nodedc-egress-error": "amd_connector_not_paired" }); + } + const result = await requestThroughAmd(target, request.method || "GET", upstreamHeaders(request), controller.signal); + upstream = result.upstream; + telemetry = result.telemetry; + if (clientAborted || controller.signal.aborted) { + upstream.destroy(); + finish({ type: "client-abort" }); + return; + } + abortUpstream = () => { + if (!upstreamEnded && !upstream.destroyed) upstream.destroy(abortError()); + }; + controller.signal.addEventListener("abort", abortUpstream, { once: true }); + if (controller.signal.aborted) abortUpstream(); + + upstream.on("data", (chunk) => { + responseBytes += chunk.length; + armBodyIdleTimer(); + }); + upstream.once("end", () => { + upstreamEnded = true; + clearBodyIdleTimer(); + }); + upstream.once("error", (error) => { + upstreamEnded = true; + clearBodyIdleTimer(); + if (!finish({ type: clientAborted ? "client-abort" : "stream-failure", errorCode: safeTransportError(error) })) return; + if (!clientAborted) { + console.warn(JSON.stringify({ event: "cesium_egress_stream_failed", host: target.hostname, error: safeTransportError(error), bytes: responseBytes })); + } + if (!response.destroyed) response.destroy(); + }); + armBodyIdleTimer(); + response.writeHead(upstream.statusCode || 502, safeResponseHeaders(upstream.headers)); + response.once("finish", () => { + downstreamFinished = true; + clearBodyIdleTimer(); + const durationMs = Date.now() - startedAt; + const status = Number(upstream.statusCode || 502); + if (!finish({ type: "response", status })) return; + // Normal tile traffic is intentionally not logged per object. Slow or + // failing responses are sufficient to diagnose VPN/transport regressions + // without creating a noisy, credential-bearing request log. + if (durationMs >= config.slowRequestMs || status >= 400) { + console.warn(JSON.stringify({ + event: "cesium_egress_slow_or_failed", + host: target.hostname, + status, + durationMs, + queueMs: telemetry.queueMs, + connectionMs: telemetry.connectionMs, + ttfbMs: telemetry.ttfbMs, + retryMs: telemetry.retryMs, + attempts: telemetry.attempts, + bytes: responseBytes, + reusedSocket: telemetry.reusedSocket, + retries: telemetry.retries, + })); + } + }); + upstream.pipe(response); + } catch (error) { + telemetry = mergeTelemetry(telemetry, error?.telemetry); + if (clientAborted || controller.signal.aborted) { + finish({ type: "client-abort" }); + return; + } + if (upstream && !upstream.destroyed) upstream.destroy(); + finish({ type: "request-failure", errorCode: safeTransportError(error) }); + error.telemetry = telemetry; + throw error; + } +} + +async function requestThroughAmd(initialTarget, method, headers, signal) { + let target = initialTarget; + let requestHeaders = { ...headers }; + let aggregate = emptyTelemetry(); + for (let redirects = 0; redirects <= 5; redirects += 1) { + let result; + try { + result = await requestWithSafeRetry(target, method, requestHeaders, signal); + } catch (error) { + error.telemetry = mergeTelemetry(aggregate, error?.telemetry); + throw error; + } + const { upstream } = result; + aggregate = mergeTelemetry(aggregate, result.telemetry); + const statusCode = Number(upstream.statusCode || 502); + const location = Array.isArray(upstream.headers.location) ? upstream.headers.location[0] : upstream.headers.location; + if ([301, 302, 303, 307, 308].includes(statusCode) && location) { + try { + aggregate.attemptMs += await drainRedirect(upstream, signal); + const redirectedTarget = parseTarget(new URL(location, target).toString()); + if (redirectedTarget.origin !== target.origin) requestHeaders = withoutAuthorization(requestHeaders); + target = redirectedTarget; + } catch (error) { + aggregate.attemptMs += Math.max(0, Number(error?.redirectDrainMs || 0)); + error.telemetry = aggregate; + throw error; + } + continue; + } + return { upstream, telemetry: aggregate }; + } + const error = typedError("amd_upstream_redirect_limit", 502); + error.telemetry = aggregate; + throw error; +} + +function withoutAuthorization(headers) { + const output = { ...headers }; + delete output.authorization; + return output; +} + +function drainRedirect(upstream, signal) { + return new Promise((resolve, reject) => { + const startedAt = Date.now(); + let finished = false; + let idleTimer; + const cleanup = () => { + if (idleTimer) clearTimeout(idleTimer); + signal.removeEventListener("abort", onAbort); + upstream.off("data", armIdleTimer); + upstream.off("end", onEnd); + upstream.off("aborted", onUpstreamAbort); + upstream.off("error", onError); + }; + const complete = () => { + if (finished) return; + finished = true; + cleanup(); + resolve(Date.now() - startedAt); + }; + const fail = (error) => { + if (finished) return; + finished = true; + cleanup(); + error.redirectDrainMs = Date.now() - startedAt; + upstream.destroy(); + reject(error); + }; + const armIdleTimer = () => { + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + const error = typedError("amd_upstream_redirect_body_idle_timeout", 502); + error.code = "ETIMEDOUT"; + fail(error); + }, config.bodyIdleTimeoutMs); + idleTimer.unref?.(); + }; + const onAbort = () => fail(abortReason(signal)); + const onEnd = () => complete(); + const onUpstreamAbort = () => { + const error = typedError("amd_upstream_redirect_aborted", 502); + error.code = "ECONNRESET"; + fail(error); + }; + const onError = (error) => fail(error); + signal.addEventListener("abort", onAbort, { once: true }); + upstream.on("data", armIdleTimer); + upstream.once("end", onEnd); + upstream.once("aborted", onUpstreamAbort); + upstream.once("error", onError); + if (signal.aborted) onAbort(); + else { + armIdleTimer(); + upstream.resume(); + } + }); +} + +async function requestWithSafeRetry(target, method, headers, signal) { + let retries = 0; + let aggregate = emptyTelemetry(); + while (true) { + try { + const result = await requestOnce(target, method, headers, signal); + aggregate = mergeTelemetry(aggregate, result.telemetry); + aggregate.retries = retries; + if (retries > 0) { + console.warn(JSON.stringify({ + event: "cesium_egress_retry_completed", + host: target.hostname, + attempt: retries + 1, + status: Number(result.upstream.statusCode || 502), + ...telemetryLogFields(result.telemetry), + totalAttemptMs: aggregate.attemptMs, + })); + } + return { upstream: result.upstream, telemetry: aggregate }; + } catch (error) { + const failedAttemptTelemetry = error?.telemetry; + aggregate = mergeTelemetry(aggregate, failedAttemptTelemetry); + if (signal.aborted || retries >= 1 || !["GET", "HEAD"].includes(method) || error?.reusedSocket !== true || !isRetryableSocketError(error)) { + if (retries > 0) { + console.warn(JSON.stringify({ + event: "cesium_egress_retry_failed", + host: target.hostname, + attempt: retries + 1, + error: safeTransportError(error), + ...telemetryLogFields(failedAttemptTelemetry), + totalAttemptMs: aggregate.attemptMs, + })); + } + error.telemetry = aggregate; + throw error; + } + retries += 1; + aggregate.retryMs += Number(error?.telemetry?.attemptMs || 0); + aggregate.retries = retries; + const errorCode = safeTransportError(error); + transport.recordRetry(target.hostname, errorCode); + console.warn(JSON.stringify({ + event: "cesium_egress_reused_socket_retry", + host: target.hostname, + attempt: retries, + error: errorCode, + ...telemetryLogFields(failedAttemptTelemetry), + })); + } + } +} + +function requestOnce(target, method, headers, externalSignal) { + return new Promise((resolve, reject) => { + const startedAt = Date.now(); + let socketAssignedAt = null; + let socketTiming = null; + let reusedSocket = false; + let settled = false; + let timedOut = false; + const pendingConnectionTiming = { startedAt: null, readyAt: null }; + const attemptController = new AbortController(); + const abortAttempt = () => { + if (!attemptController.signal.aborted) attemptController.abort(abortReason(externalSignal)); + }; + if (externalSignal.aborted) abortAttempt(); + else externalSignal.addEventListener("abort", abortAttempt, { once: true }); + const client = https.request({ + protocol: "https:", + hostname: target.hostname, + port: 443, + method, + path: `${target.pathname || "/"}${target.search || ""}`, + headers, + agent: transport.agent, + signal: attemptController.signal, + nodedcAbortSignal: attemptController.signal, + nodedcConnectionTiming: pendingConnectionTiming, + }); + transport.observeQueuePeak(); + queueMicrotask(() => transport.observeQueuePeak()); + const timeoutError = typedError("amd_upstream_timeout", 502); + timeoutError.code = "ETIMEDOUT"; + const timeout = setTimeout(() => { + timedOut = true; + if (!attemptController.signal.aborted) attemptController.abort(timeoutError); + }, config.upstreamTimeoutMs); + timeout.unref?.(); + const cleanup = () => { + clearTimeout(timeout); + externalSignal.removeEventListener("abort", abortAttempt); + }; + const attemptTelemetry = (endedAt, receivedHeaders, error) => { + const failedConnectionTiming = error?.[connectionTimingSymbol]; + const timing = socketTiming || failedConnectionTiming || (pendingConnectionTiming.startedAt ? pendingConnectionTiming : null); + const newConnection = !reusedSocket && timing; + const queueEnd = newConnection?.startedAt || socketAssignedAt || endedAt; + const connectionMs = newConnection ? Math.max(0, Number(newConnection.readyAt || endedAt) - Number(newConnection.startedAt || endedAt)) : 0; + return { + queueMs: Math.max(0, Number(queueEnd) - startedAt), + connectionMs, + ttfbMs: receivedHeaders && socketAssignedAt ? Math.max(0, endedAt - socketAssignedAt) : 0, + attemptMs: Math.max(0, endedAt - startedAt), + retryMs: 0, + attempts: 1, + retries: 0, + reusedSocket, + }; + }; + client.once("socket", (socket) => { + socketAssignedAt = Date.now(); + socketTiming = socket[connectionTimingSymbol] || null; + const priorAssignments = Number(socketTiming?.assignments || 0); + reusedSocket = client.reusedSocket === true || priorAssignments > 0; + if (socketTiming) socketTiming.assignments = priorAssignments + 1; + }); + client.once("error", (rawError) => { + if (settled) return; + settled = true; + cleanup(); + const error = timedOut && !externalSignal.aborted ? timeoutError : externalSignal.aborted ? abortReason(externalSignal) : rawError; + if (rawError?.[connectionTimingSymbol] && !error[connectionTimingSymbol]) error[connectionTimingSymbol] = rawError[connectionTimingSymbol]; + error.reusedSocket = reusedSocket || client.reusedSocket === true; + error.telemetry = attemptTelemetry(Date.now(), false, error); + reject(error); + }); + client.once("response", (upstream) => { + if (settled) { + upstream.destroy(); + return; + } + settled = true; + cleanup(); + const endedAt = Date.now(); + resolve({ + upstream, + telemetry: attemptTelemetry(endedAt, true), + }); + }); + client.end(); + transport.observeQueuePeak(); + }); +} + +function emptyTelemetry() { + return { queueMs: 0, connectionMs: 0, ttfbMs: 0, attemptMs: 0, retryMs: 0, attempts: 0, retries: 0, reusedSocket: false }; +} + +function telemetryLogFields(telemetry) { + if (!telemetry) return {}; + return { + queueMs: Math.max(0, Number(telemetry.queueMs || 0)), + connectionMs: Math.max(0, Number(telemetry.connectionMs || 0)), + ttfbMs: Math.max(0, Number(telemetry.ttfbMs || 0)), + attemptMs: Math.max(0, Number(telemetry.attemptMs || 0)), + retryMs: Math.max(0, Number(telemetry.retryMs || 0)), + attempts: Math.max(0, Number(telemetry.attempts || 0)), + retries: Math.max(0, Number(telemetry.retries || 0)), + reusedSocket: telemetry.reusedSocket === true, + }; +} + +function mergeTelemetry(left, right) { + const merged = { ...emptyTelemetry(), ...(left || {}) }; + if (!right) return merged; + for (const name of ["queueMs", "connectionMs", "ttfbMs", "attemptMs", "retryMs", "attempts", "retries"]) { + if (name === "retries") merged[name] = Math.max(Number(merged[name] || 0), Number(right[name] || 0)); + else merged[name] += Math.max(0, Number(right[name] || 0)); + } + merged.reusedSocket ||= right.reusedSocket === true; + return merged; +} + +function createTransport(runtimeConfig) { + const metrics = { + openedTunnels: 0, + tunnelFailures: 0, + requests: 0, + terminalRequests: 0, + inFlightRequests: 0, + completedResponses: 0, + failedRequests: 0, + slowRequests: 0, + responseBytes: 0, + partialBytes: 0, + reusedSocketRequests: 0, + retries: 0, + streamFailures: 0, + clientAborts: 0, + maxQueuedRequests: 0, + timing: { + queueMs: { total: 0, max: 0 }, + connectionMs: { total: 0, max: 0 }, + ttfbMs: { total: 0, max: 0 }, + attemptMs: { total: 0, max: 0 }, + retryMs: { total: 0, max: 0 }, + durationMs: { total: 0, max: 0 }, + }, + byHost: {}, + lastFailure: null, + lastFailureAt: null, + }; + const agent = new https.Agent({ + keepAlive: true, + keepAliveMsecs: 30_000, + maxSockets: runtimeConfig.poolMaxSockets, + maxFreeSockets: runtimeConfig.poolMaxFreeSockets, + scheduling: "lifo", + }); + // https.Agent maintains a separate pool per origin. A Cesium/Bing tile + // therefore reuses an already authenticated NAS -> AMD -> VPN -> TLS path + // for its host instead of paying a fresh CONNECT and TLS handshake. + agent.createConnection = (options, callback) => { + const hostname = String(options.servername || options.hostname || options.host || "").toLowerCase(); + const signal = options.nodedcAbortSignal; + const pendingConnectionTiming = options.nodedcConnectionTiming; + void (async () => { + if (!allowedHosts.has(hostname)) throw typedError("cesium_target_not_allowed", 403); + const connectorToken = await readConnectorToken(); + if (!connectorToken) throw typedError("amd_connector_not_paired", 503); + const startedAt = Date.now(); + if (pendingConnectionTiming) pendingConnectionTiming.startedAt = startedAt; + let connectedAt = startedAt; + try { + const tunnel = await openConnectorTunnel({ hostname }, connectorToken, signal); + connectedAt = Date.now(); + const secureSocket = await openTlsTunnel(tunnel, hostname, signal); + const readyAt = Date.now(); + if (pendingConnectionTiming) pendingConnectionTiming.readyAt = readyAt; + secureSocket[connectionTimingSymbol] = { startedAt, readyAt, assignments: 0 }; + metrics.openedTunnels += 1; + console.log(JSON.stringify({ + event: "cesium_egress_tunnel_opened", + host: hostname, + connectorMs: connectedAt - startedAt, + tlsMs: readyAt - connectedAt, + })); + callback(null, secureSocket); + } catch (error) { + const readyAt = Date.now(); + if (pendingConnectionTiming) pendingConnectionTiming.readyAt = readyAt; + error[connectionTimingSymbol] = { startedAt, readyAt, assignments: 0 }; + throw error; + } + })().catch((error) => { + const errorCode = safeError(error); + if (error?.code !== "ABORT_ERR") { + metrics.tunnelFailures += 1; + metrics.lastFailure = errorCode; + metrics.lastFailureAt = new Date().toISOString(); + } + callback(error); + }); + return undefined; + }; + + return { + agent, + close(callback) { agent.destroy(); callback(); }, + beginRequest(host) { + const hostMetrics = perHostMetrics(metrics, host); + metrics.requests += 1; + hostMetrics.requests += 1; + metrics.inFlightRequests += 1; + hostMetrics.inFlightRequests += 1; + let finished = false; + return { + complete({ type, status = 0, durationMs = 0, bytes = 0, telemetry = emptyTelemetry(), errorCode = "" }) { + if (finished) return false; + finished = true; + metrics.terminalRequests += 1; + metrics.inFlightRequests = Math.max(0, metrics.inFlightRequests - 1); + hostMetrics.terminalRequests += 1; + hostMetrics.inFlightRequests = Math.max(0, hostMetrics.inFlightRequests - 1); + const safeBytes = Math.max(0, Number(bytes || 0)); + if (telemetry.reusedSocket) { + metrics.reusedSocketRequests += 1; + hostMetrics.reusedSocketRequests += 1; + } + observe(metrics.timing.queueMs, telemetry.queueMs); + observe(metrics.timing.connectionMs, telemetry.connectionMs); + observe(metrics.timing.ttfbMs, telemetry.ttfbMs); + observe(metrics.timing.attemptMs, telemetry.attemptMs); + observe(metrics.timing.retryMs, telemetry.retryMs); + observe(metrics.timing.durationMs, durationMs); + if (type === "response") { + metrics.completedResponses += 1; + hostMetrics.completedResponses += 1; + metrics.responseBytes += safeBytes; + hostMetrics.bytes += safeBytes; + } else { + metrics.partialBytes += safeBytes; + hostMetrics.partialBytes += safeBytes; + } + if (type === "client-abort") { + metrics.clientAborts += 1; + hostMetrics.clientAborts += 1; + } + const failure = type === "stream-failure" || type === "request-failure" || type === "response" && Number(status) >= 400; + if (failure) { + metrics.failedRequests += 1; + hostMetrics.failedRequests += 1; + if (type === "stream-failure") { + metrics.streamFailures += 1; + hostMetrics.streamFailures += 1; + } + metrics.lastFailure = errorCode || `upstream_http_${status}`; + metrics.lastFailureAt = new Date().toISOString(); + hostMetrics.lastTransportError = errorCode || `upstream_http_${status}`; + } + if (type !== "client-abort" && durationMs >= runtimeConfig.slowRequestMs) metrics.slowRequests += 1; + return true; + }, + }; + }, + recordRetry(host, errorCode) { + metrics.retries += 1; + const hostMetrics = perHostMetrics(metrics, host); + hostMetrics.retries += 1; + hostMetrics.lastTransportError = errorCode; + }, + observeQueuePeak() { + const queuedRequests = socketCount(agent.requests); + metrics.maxQueuedRequests = Math.max(metrics.maxQueuedRequests, queuedRequests); + }, + status() { + const queuedRequests = socketCount(agent.requests); + metrics.maxQueuedRequests = Math.max(metrics.maxQueuedRequests, queuedRequests); + return { + pool: { + activeSockets: socketCount(agent.sockets), + idleSockets: socketCount(agent.freeSockets), + queuedRequests, + maxQueuedRequests: metrics.maxQueuedRequests, + maxSockets: runtimeConfig.poolMaxSockets, + maxFreeSockets: runtimeConfig.poolMaxFreeSockets, + byOrigin: poolByOrigin(agent), + }, + metrics: publicTransportMetrics(metrics), + }; + }, + }; +} + +function perHostMetrics(metrics, host) { + const safeHost = allowedHosts.has(String(host || "").toLowerCase()) ? String(host).toLowerCase() : "unknown"; + metrics.byHost[safeHost] ||= { + requests: 0, + terminalRequests: 0, + inFlightRequests: 0, + completedResponses: 0, + failedRequests: 0, + reusedSocketRequests: 0, + retries: 0, + streamFailures: 0, + clientAborts: 0, + bytes: 0, + partialBytes: 0, + lastTransportError: null, + }; + return metrics.byHost[safeHost]; +} + +function observe(bucket, rawValue) { + const value = Math.max(0, Number(rawValue || 0)); + bucket.total += value; + bucket.max = Math.max(bucket.max, value); +} + +function publicTransportMetrics(metrics) { + const average = (bucket) => metrics.terminalRequests ? Math.round(bucket.total / metrics.terminalRequests) : 0; + return { + openedTunnels: metrics.openedTunnels, + tunnelFailures: metrics.tunnelFailures, + requests: metrics.requests, + terminalRequests: metrics.terminalRequests, + inFlightRequests: metrics.inFlightRequests, + completedResponses: metrics.completedResponses, + failedRequests: metrics.failedRequests, + slowRequests: metrics.slowRequests, + responseBytes: metrics.responseBytes, + partialBytes: metrics.partialBytes, + reusedSocketRequests: metrics.reusedSocketRequests, + retries: metrics.retries, + streamFailures: metrics.streamFailures, + clientAborts: metrics.clientAborts, + timing: { + queueMs: { average: average(metrics.timing.queueMs), max: metrics.timing.queueMs.max }, + connectionMs: { average: average(metrics.timing.connectionMs), max: metrics.timing.connectionMs.max }, + ttfbMs: { average: average(metrics.timing.ttfbMs), max: metrics.timing.ttfbMs.max }, + attemptMs: { average: average(metrics.timing.attemptMs), max: metrics.timing.attemptMs.max }, + retryMs: { average: average(metrics.timing.retryMs), max: metrics.timing.retryMs.max }, + durationMs: { average: average(metrics.timing.durationMs), max: metrics.timing.durationMs.max }, + }, + byHost: metrics.byHost, + lastFailure: metrics.lastFailure, + lastFailureAt: metrics.lastFailureAt, + }; +} + +function poolByOrigin(agent) { + const origins = {}; + for (const [key, sockets] of Object.entries(agent.sockets)) poolOrigin(origins, key).active += sockets.length; + for (const [key, sockets] of Object.entries(agent.freeSockets)) poolOrigin(origins, key).idle += sockets.length; + for (const [key, requests] of Object.entries(agent.requests)) poolOrigin(origins, key).queued += requests.length; + return origins; +} + +function poolOrigin(origins, agentKey) { + const host = String(agentKey || "").split(":")[0].toLowerCase(); + const safeHost = allowedHosts.has(host) ? host : "unknown"; + origins[safeHost] ||= { active: 0, idle: 0, queued: 0 }; + return origins[safeHost]; +} + +function openConnectorTunnel(target, connectorToken, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(abortReason(signal)); + return; + } + const socket = connectNet({ host: config.connectorHost, port: config.connectorPort }); + let buffer = Buffer.alloc(0); + let finished = false; + const timeoutError = typedError("amd_connector_timeout", 502); + timeoutError.code = "ETIMEDOUT"; + const timeout = setTimeout(() => fail(timeoutError), config.connectTimeoutMs); + timeout.unref?.(); + const cleanup = () => { + clearTimeout(timeout); + signal?.removeEventListener("abort", onAbort); + socket.off("connect", onConnect); + socket.off("error", fail); + socket.off("data", onData); + }; + const finish = (value) => { + if (finished) return; + finished = true; + cleanup(); + resolve(value); + }; + const fail = (error) => { + if (finished) return; + finished = true; + cleanup(); + socket.destroy(); + reject(error); + }; + const onAbort = () => fail(abortReason(signal)); + const onConnect = () => socket.write(`CONNECT ${target.hostname}:443 HTTP/1.1\r\nHost: ${target.hostname}:443\r\nProxy-Authorization: Bearer ${connectorToken}\r\n\r\n`); + const onData = (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + if (buffer.length > 16 * 1024) return fail(typedError("amd_connector_response_invalid", 502)); + const end = buffer.indexOf("\r\n\r\n"); + if (end < 0) return; + if (!/^HTTP\/1\.[01] 200\b/.test(buffer.subarray(0, end).toString("ascii"))) return fail(typedError("amd_connector_rejected", 502)); + const remainder = buffer.subarray(end + 4); + if (remainder.length) socket.unshift(remainder); + finish(socket); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + socket.once("connect", onConnect); + socket.on("error", fail); + socket.on("data", onData); + }); +} + +function openTlsTunnel(socket, hostname, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + socket.destroy(); + reject(abortReason(signal)); + return; + } + const secureSocket = connectTls({ socket, servername: hostname, ALPNProtocols: ["http/1.1"], rejectUnauthorized: !config.allowInsecureTls }); + let finished = false; + const timeoutError = typedError("amd_upstream_tls_timeout", 502); + timeoutError.code = "ETIMEDOUT"; + const timeout = setTimeout(() => fail(timeoutError), config.connectTimeoutMs); + timeout.unref?.(); + const cleanup = () => { + clearTimeout(timeout); + signal?.removeEventListener("abort", onAbort); + secureSocket.off("secureConnect", finish); + secureSocket.off("error", fail); + }; + const finish = () => { + if (finished) return; + finished = true; + cleanup(); + resolve(secureSocket); + }; + const fail = (error) => { + if (finished) return; + finished = true; + cleanup(); + secureSocket.destroy(); + reject(error); + }; + const onAbort = () => fail(abortReason(signal)); + signal?.addEventListener("abort", onAbort, { once: true }); + secureSocket.once("secureConnect", finish); + secureSocket.once("error", fail); + }); +} + +function upstreamHeaders(request) { + // Do not force `Connection: close`: the shared Agent owns a deliberately + // bounded keep-alive pool for the approved upstream hosts. + const headers = { accept: safeHeader(request.headers.accept, "*/*"), "accept-encoding": "identity" }; + for (const name of ["range", "if-none-match", "if-modified-since"]) { + const value = safeHeader(request.headers[name], ""); + if (value) headers[name] = value; + } + const authorization = safeHeader(request.headers["x-nodedc-cesium-authorization"], ""); + if (authorization) headers.authorization = authorization; + const referer = safeHeader(request.headers["x-nodedc-map-referer"], ""); + if (referer) headers.referer = referer; + return headers; +} + +function safeResponseHeaders(headers) { + const output = {}; + const skipped = new Set(["connection", "keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade"]); + for (const [name, raw] of Object.entries(headers)) { + if (skipped.has(name.toLowerCase()) || raw === undefined) continue; + const value = Array.isArray(raw) ? raw.join(", ") : String(raw); + if (value.length <= 8192 && !/[\r\n\u0000]/.test(value)) output[name] = value; + } + return output; +} + +function parseTarget(value) { + let target; + try { target = new URL(String(value)); } catch { throw typedError("cesium_target_invalid", 400); } + if (target.protocol !== "https:" || target.username || target.password || target.port && target.port !== "443" || !allowedHosts.has(target.hostname.toLowerCase())) throw typedError("cesium_target_not_allowed", 403); + target.hostname = target.hostname.toLowerCase(); + return target; +} + +async function readConnectorToken() { + try { + const token = (await readFile(config.connectorTokenFile, "utf8")).trim(); + return isSecret(token) ? token : null; + } catch (error) { + if (error?.code === "ENOENT") return null; + throw typedError("connector_pair_state_unreadable", 500); + } +} + +async function persistConnectorToken(token) { + await mkdir(dirname(config.connectorTokenFile), { recursive: true, mode: 0o700 }); + const temporary = `${config.connectorTokenFile}.${process.pid}.${Date.now()}.tmp`; + let handle; + try { + handle = await open(temporary, "wx", 0o600); + await handle.writeFile(`${token}\n`, "ascii"); + await handle.sync(); + await handle.close(); + handle = null; + // link(2) is the compare-and-set: unlike rename(), it cannot overwrite an + // existing pair state. Two concurrent pair attempts therefore produce one + // durable secret and one harmless 409 response. + try { + await link(temporary, config.connectorTokenFile); + } catch (error) { + if (error?.code === "EEXIST") throw typedError("already_paired", 409); + throw error; + } + } finally { + if (handle) await handle.close(); + await unlink(temporary).catch(() => {}); + } +} + +async function readSecretFile(path, label) { + let value; + try { value = (await readFile(String(path), "utf8")).trim(); } catch { throw new Error(`${label}_unreadable`); } + if (!value || value.length > 4096 || /[\u0000-\u001f\u007f\s]/.test(value)) throw new Error(`${label}_invalid`); + return value; +} + +function isMapAuthorized(raw) { + if (Array.isArray(raw)) return false; + const candidate = Buffer.from(String(raw || ""), "utf8"); + return candidate.length === config.mapToken.length && timingSafeEqual(candidate, config.mapToken); +} + +function readJsonBody(request, maxBytes) { + return new Promise((resolve, reject) => { + const chunks = []; + let bytes = 0; + request.on("data", (chunk) => { + bytes += chunk.length; + if (bytes > maxBytes) return reject(typedError("pair_payload_too_large", 413)); + chunks.push(chunk); + }); + request.once("error", reject); + request.once("end", () => { + try { resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); } catch { reject(typedError("pair_payload_invalid", 400)); } + }); + }); +} + +function writeJson(response, status, body, extraHeaders = {}) { + const payload = JSON.stringify(body); + response.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(payload), + "cache-control": "no-store", + ...extraHeaders, + }); + response.end(payload); +} + +function parsePort(raw, fallback) { + const value = Number(String(raw || fallback).trim()); + if (!Number.isInteger(value) || value < 1024 || value > 65535) throw new Error("invalid_port"); + return value; +} + +function parseDuration(raw, fallbackSeconds) { + const value = Number(String(raw || fallbackSeconds).trim()); + if (!Number.isInteger(value) || value < 1 || value > 120) throw new Error("invalid_duration"); + return value * 1000; +} + +function parsePoolSize(raw, fallback) { + const value = Number(String(raw || fallback).trim()); + if (!Number.isInteger(value) || value < 1 || value > 32) throw new Error("invalid_pool_size"); + return value; +} + +function socketCount(table) { + return Object.values(table).reduce((total, sockets) => total + (Array.isArray(sockets) ? sockets.length : 0), 0); +} + +function parseIpv4(raw, label) { + const value = String(raw || "").trim(); + const octets = value.split(".").map(Number); + if (octets.length !== 4 || octets.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) throw new Error(`${label}_invalid`); + return value; +} + +function normalizeAddress(value) { return String(value || "").replace(/^::ffff:/, ""); } +function safeHeader(raw, fallback) { + if (Array.isArray(raw)) return fallback; + const value = String(raw || "").trim(); + return value.length <= 4096 && !/[\r\n\u0000]/.test(value) ? value : fallback; +} +function parseBoolean(value, fallback) { return value === undefined || value === "" ? fallback : ["1", "true", "yes", "on"].includes(String(value).trim().toLowerCase()); } +function isSecret(value) { return /^[A-Za-z0-9_-]{48,256}$/.test(value); } +function typedError(message, statusCode) { const error = new Error(message); error.statusCode = statusCode; return error; } +function abortError() { const error = typedError("amd_client_aborted", 499); error.code = "ABORT_ERR"; return error; } +function abortReason(signal) { return signal?.reason instanceof Error ? signal.reason : abortError(); } +function safeError(error) { return String(error?.message || "dc_amd_proxy_error").replace(/[^A-Za-z0-9_.:-]/g, "_").slice(0, 120); } +function isRetryableSocketError(error) { return new Set(["ECONNRESET", "EPIPE", "ETIMEDOUT", "ECONNABORTED"]).has(String(error?.code || "").toUpperCase()); } +function safeTransportError(error) { + const code = String(error?.code || "").trim().toLowerCase(); + if (/^[a-z0-9_]{1,48}$/.test(code)) return `amd_upstream_${code}`; + return safeError(error); +} diff --git a/services/map-gateway/Dockerfile b/services/map-gateway/Dockerfile index 3ee5406..726ab22 100644 --- a/services/map-gateway/Dockerfile +++ b/services/map-gateway/Dockerfile @@ -10,11 +10,9 @@ RUN apk add --no-cache su-exec COPY package.json ./ COPY src ./src -COPY docker-entrypoint.sh /usr/local/bin/nodedc-map-gateway - -RUN chmod +x /usr/local/bin/nodedc-map-gateway +COPY runtime-entrypoint.mjs /app/runtime-entrypoint.mjs EXPOSE 18103 -ENTRYPOINT ["/usr/local/bin/nodedc-map-gateway"] +ENTRYPOINT ["node", "/app/runtime-entrypoint.mjs"] CMD ["node", "src/server.mjs"] diff --git a/services/map-gateway/README.md b/services/map-gateway/README.md index e51c364..a466acc 100644 --- a/services/map-gateway/README.md +++ b/services/map-gateway/README.md @@ -4,14 +4,14 @@ ## Что делает -- хранит master `CESIUM_ION_TOKEN` только в process environment; -- выдаёт браузеру только asset-scoped endpoint token для allowlisted ion assets; +- хранит master Cesium Ion token только в private runtime storage: первоначально он может прийти из server environment, а после admin rotation — из закрытого файла в NAS volume; +- никогда не выдаёт browser-у master, asset-scoped Cesium token или Bing key: endpoint response содержит только публичный provider URL, а Gateway подставляет credential в свой upstream request; - проксирует и кэширует tile/3D Tiles/terrain resources через `GET /api/map/cache?url=`; - может отдать перенесённый локальный Engine snapshot через `MAP_GATEWAY_LEGACY_CACHE_HOSTS`, но никогда не догружает для него cache miss из сети; - работает с persistent volume, а не с Git или Docker image layer; - при upstream outage отдаёт ранее сохранённый cache object; offline режим включается только для providers, явно разрешённых их лицензией; - защищает proxy allowlist-ом upstream hosts, HTTPS-only правилом, лимитами размера и token-free cache key; -- ведёт лёгкий persistent cache index для LRU eviction и health/stats. +- ведёт лёгкий persistent cache index и health/stats без автоматического удаления уже записанных tiles. ## API @@ -21,23 +21,31 @@ GET /api/map/ion/assets/:assetId/endpoint GET|HEAD /api/map/cache?url= ``` -`/api/map/ion/assets/:assetId/endpoint` разрешает только `CESIUM_ION_ASSET_ALLOWLIST`. Ответ содержит runtime asset token, а не master token. Browser использует endpoint вместе с Cesium `DefaultProxy`, направляющим resource requests в `/api/map/cache`. +Внутренний admin route `GET|PUT /api/map/admin/cesium-ion` не является Browser API: он доступен только из Foundry по HMAC. Ключ подписи создаётся и хранится root-owned deploy runner в `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`, а оба сервиса получают только read-only file mount. Он не является `.env`-значением, настройкой Foundry или Cesium credential. Перед записью `PUT` проверяет candidate token через Cesium asset endpoints `1` (terrain), `2` (imagery) и `96188` (3D Buildings). Не прошедшее проверку значение не заменяет рабочее. Успешный token записывается атомарно в `${MAP_CACHE_DIR}/secrets/cesium-ion-token` с mode `0600`; ответ содержит только факт конфигурации, safe verification state и audit metadata — никогда само значение. -Ion endpoint metadata также сохраняется в persistent volume с файловыми правами `0600`. Это нужно для cold start в offline: Cesium получает сохранённый asset URL и asset-scoped token, а proxy отвечает только из ранее записанного cache. В metadata никогда не записывается master token. +`/api/map/ion/assets/:assetId/endpoint` разрешает только `CESIUM_ION_ASSET_ALLOWLIST`. Ответ не содержит credential: Browser использует публичный provider URL вместе с Cesium `DefaultProxy`, направляющим resource requests в `/api/map/cache`. Gateway валидирует scope URL, удаляет любые credential query parameters из browser request и добавляет соответствующий server-side credential только перед обращением к provider. + +Ion endpoint metadata с asset credential сохраняется только в private persistent storage с файловыми правами `0600`. Это нужно для cold start и offline: Browser по-прежнему получает только sanitised URL, а Gateway отвечает из ранее записанного cache. В metadata никогда не записывается master token; NAS backup этого каталога считается service-sensitive. ## Cache modes -- `readwrite` — свежий cache hit отдаётся сразу; stale object обновляется online; при upstream failure возвращается stale copy; -- `readonly` — использует cache, но не записывает новый; +- `readwrite` — при включённом `Не перезаписывать cache` отдаёт уже записанный object прямо из общего TileCache; только miss идёт к официальному upstream и атомарно дописывается. Явный `refresh` viewport получает live response и заменяет объект; +- `readonly` — отдаёт ранее записанный object из TileCache; новый object через Gateway может быть показан live, но не записывается; - `offline` — никогда не идёт наружу; cache miss возвращает `504`. +`nodedc_cache_refresh=1` — единственный путь заменить уже записанный объект. Без него существующий object всегда отдаётся прямо с NAS и вообще не обращается к Ion/AMD/VPN. Foundry создаёт refresh либо когда Application явно разрешил обновление cache, либо для одноразового действия «Обновить текущий viewport». При достижении `MAP_CACHE_MAX_MB` Gateway не удаляет LRU-объекты: новый miss продолжает live-маршрут без записи с `x-nodedc-map-cache: live-pass-through-cache-full`. + +Первый cold miss начинает стримиться клиенту сразу после provider headers и одновременно записывается в private temp-файл. Только полностью полученный object публикуется через atomic rename и durable `index.json`; partial/error никогда не становится cache entry. Одинаковые параллельные misses делят один upstream download, а отмена одного browser request не прерывает server-owned fill для других инстансов. Warm hit не меняет index. Параллельные новые объекты объединяются в минимальное число полных index snapshots вместо одного O(N) rewrite на каждый tile. + `offline` дополнительно требует, чтобы hostname был явно указан в `MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST`. По умолчанию список пуст: Gateway не превращает Cesium ion или public OSM tile service в offline distribution. Это осознанная provider policy, а не техническая ошибка cache. Range requests пока transparently проксируются и не записываются как partial cache object. Полные GET responses кэшируются. Это безопасная начальная граница для terrain/3D Tiles; отдельным следующим шагом добавляется range-aware chunk cache после замеров реальных GOS/3D Tiles assets. ## Persistent storage -Platform Compose использует два внешних named volume: mutable `nodedc-platform_map-live-tile-cache` монтируется в `/var/lib/nodedc-map-live-cache`, read-only snapshot `nodedc-platform_map-offline-snapshot` — в `/var/lib/nodedc-map-offline-snapshot`. Их содержимое намеренно исключено из Git. Внешний volume не удаляется через `docker compose down -v`; очистка требует явного `docker volume rm` при остановленном Gateway. При необходимости offline seed доставляется отдельным export/import artifact, а не коммитом runtime cache. +Production Platform Compose использует два NAS host directories: mutable `/volume1/docker/nodedc-platform/map-gateway/live-tile-cache` монтируется в `/var/lib/nodedc-map-live-cache`, read-only `/volume1/docker/nodedc-platform/map-gateway/offline-snapshot` — в `/var/lib/nodedc-map-offline-snapshot`. Их создаёт root-owned deploy runner при первом Map Gateway rollout. Они намеренно исключены из Git и artifact; Compose teardown их не удаляет. При необходимости offline seed доставляется отдельным export/import artifact, а не коммитом runtime cache. + +Внутри mutable volume имеется service-sensitive подкаталог `secrets/`: там лежит только текущий Cesium Ion master token и metadata последнего изменения. Это не TileCache, не часть offline export и не объект SMB-операций. Gateway создаёт его с directory mode `0700`, files `0600`; не включать его в backup, export или diagnostic archive без отдельной процедуры секретов. Кэш не способен создать участок карты, который никогда не был скачан. Для целевых офлайн-регионов нужен отдельный licensed dataset/provider и его `seed/prefetch` job: он заранее проходит разрешённые assets, zoom-диапазоны и географические области, записывает их в тот же persistent volume и формирует проверяемый export/import artifact. Эта задача не смешивается с runtime gateway и не помещает binary tiles в историю репозитория. @@ -66,9 +74,15 @@ npm run import:engine-cache -- /absolute/path/to/nodedc-data/api/cesium/manifest ## Required environment ```text +# Optional migration bootstrap; the private token file is preferred after an +# admin has configured it through Foundry. CESIUM_ION_TOKEN= -CESIUM_ION_ASSET_ALLOWLIST=1,96188 -MAP_GATEWAY_UPSTREAM_ALLOWLIST=api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org +# Production: runner-owned NODEDC_MAP_GATEWAY_ADMIN_SECRET_FILE is mounted +# automatically. Do not put the secret into an env file. +CESIUM_ION_ASSET_ALLOWLIST=1,2,96188 +CESIUM_ION_ENDPOINT_TTL_SECONDS=300 +CESIUM_ION_ENDPOINT_REFRESH_AHEAD_SECONDS=60 +MAP_GATEWAY_UPSTREAM_ALLOWLIST=api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net MAP_GATEWAY_LEGACY_CACHE_HOSTS=ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST= MAP_CACHE_DIR=/var/lib/nodedc-map-cache @@ -76,12 +90,13 @@ MAP_CACHE_MODE=readwrite ``` # Cache topology -`MAP_CACHE_DIR` is the mutable **live** cache. In the Platform compose profile it -is mounted as `map-live-tile-cache` and is the only store that can receive new -upstream objects. +`MAP_CACHE_DIR` is the mutable **live** cache. In the production Platform +compose profile it is bind-mounted from the NAS path +`/volume1/docker/nodedc-platform/map-gateway/live-tile-cache` and is the only +store that can receive new upstream objects. `MAP_OFFLINE_SNAPSHOT_DIR` is optional and mounted read-only as -`map-offline-snapshot`. It is populated only by the controlled Engine cache -import. Requests marked with the `offline` cache profile are served exclusively -from this snapshot; a missing tile returns a cache miss and never reaches an -upstream provider. +`/volume1/docker/nodedc-platform/map-gateway/offline-snapshot`. It is +populated only by the controlled Engine cache import. Requests marked with the +`offline` cache profile are served exclusively from this snapshot; a missing +tile returns a cache miss and never reaches an upstream provider. diff --git a/services/map-gateway/docker-entrypoint.sh b/services/map-gateway/docker-entrypoint.sh deleted file mode 100644 index b3d913d..0000000 --- a/services/map-gateway/docker-entrypoint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -eu - -# Docker creates a fresh named volume as root. Prepare only the mutable live -# cache before dropping privileges; the imported offline snapshot stays read-only. -if [ -d "${MAP_CACHE_DIR:-}" ]; then - mkdir -p "$MAP_CACHE_DIR" - chown -R node:node "$MAP_CACHE_DIR" -fi - -exec su-exec node "$@" diff --git a/services/map-gateway/package.json b/services/map-gateway/package.json index 1ccbef8..73bac8a 100644 --- a/services/map-gateway/package.json +++ b/services/map-gateway/package.json @@ -6,6 +6,10 @@ "scripts": { "start": "node src/server.mjs", "dev": "node --watch src/server.mjs", + "test:credential-boundary": "node scripts/smoke-no-browser-credentials.mjs", + "test:admin-token-boundary": "node scripts/smoke-admin-boundary.mjs", + "test:live-cache-fallback": "node scripts/smoke-live-cache-fallback.mjs", + "test:streaming-cache-fill": "node scripts/smoke-streaming-cache-fill.mjs", "audit:engine-cache": "node scripts/audit-engine-cache.mjs", "import:engine-cache": "node scripts/import-engine-cache.mjs" }, diff --git a/services/map-gateway/runtime-entrypoint.mjs b/services/map-gateway/runtime-entrypoint.mjs new file mode 100644 index 0000000..af7bd2e --- /dev/null +++ b/services/map-gateway/runtime-entrypoint.mjs @@ -0,0 +1,17 @@ +import { spawn } from "node:child_process"; +import { mkdir, chown } from "node:fs/promises"; + +const cacheDir = String(process.env.MAP_CACHE_DIR || "").trim(); +if (cacheDir) { + await mkdir(cacheDir, { recursive: true }); + await chown(cacheDir, 1000, 1000); +} +if (process.getuid?.() === 0) { + process.setgid(1000); + process.setuid(1000); +} +const [command, ...args] = process.argv.slice(2); +if (!command) throw new Error("map_gateway_runtime_command_required"); +const child = spawn(command, args, { stdio: "inherit" }); +child.on("exit", (code, signal) => process.exitCode = code ?? (signal ? 1 : 0)); +child.on("error", (error) => { throw error; }); diff --git a/services/map-gateway/scripts/smoke-admin-boundary.mjs b/services/map-gateway/scripts/smoke-admin-boundary.mjs new file mode 100644 index 0000000..93780db --- /dev/null +++ b/services/map-gateway/scripts/smoke-admin-boundary.mjs @@ -0,0 +1,366 @@ +import assert from "node:assert/strict"; +import { createHash, createHmac } from "node:crypto"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createServer as createHttpServer } from "node:http"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; + +const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-admin-smoke-")); +const cacheDir = join(root, "cache"); +const port = await freePort(); +const egressPort = await freePort(); +const adminSecret = "gateway-admin-smoke-secret"; +const providerToken = "cesium-private-token-should-never-be-returned"; +const rotatedProviderToken = "rotated-cesium-private-token-should-never-be-returned"; +const egressProxyToken = "map-egress-private-token-should-never-be-returned"; +const ionReferer = "https://foundry.example.test/page-library/"; +const providerState = createProviderState(); +let child; +let egress; +let childOutput = ""; + +try { + const egressTokenFile = join(root, "map-egress-proxy-token"); + await writeFile(egressTokenFile, `${egressProxyToken}\n`, { mode: 0o600 }); + egress = createCesiumEgressMock(providerState, egressProxyToken); + egress.listen(egressPort, "127.0.0.1"); + await once(egress, "listening"); + child = spawn(process.execPath, ["src/server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + PORT: String(port), + MAP_CACHE_DIR: cacheDir, + MAP_GATEWAY_ALLOW_ANONYMOUS: "true", + MAP_CACHE_MODE: "readwrite", + NODE_ENV: "test", + CESIUM_ION_TOKEN: "", + // A malformed legacy override may add an approved asset, but must never + // remove the three canonical Foundry map assets. + CESIUM_ION_ASSET_ALLOWLIST: "999", + CESIUM_ION_API_BASE_URL: "https://api.cesium.com/", + MAP_GATEWAY_UPSTREAM_ALLOWLIST: "api.cesium.com,assets.ion.cesium.com,dev.virtualearth.net", + MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK: "true", + MAP_GATEWAY_EGRESS_URL: `http://127.0.0.1:${egressPort}/`, + MAP_GATEWAY_EGRESS_PROXY_TOKEN_FILE: egressTokenFile, + NODEDC_MAP_GATEWAY_ADMIN_SECRET: adminSecret, + CESIUM_ION_ENDPOINT_TTL_SECONDS: "2", + CESIUM_ION_ENDPOINT_REFRESH_AHEAD_SECONDS: "1", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.on("data", (chunk) => { childOutput += String(chunk); }); + child.stderr.on("data", (chunk) => { childOutput += String(chunk); }); + + await waitForGateway(port, child); + const path = "/api/map/admin/cesium-ion"; + const unsigned = await fetch(`http://127.0.0.1:${port}${path}`); + assert.equal(unsigned.status, 401); + + const initial = await requestAdmin(port, adminSecret, "GET", path, "user_root", "", ionReferer); + assert.equal(initial.response.status, 200); + assert.equal(initial.body.configured, false); + + const saved = await requestAdmin(port, adminSecret, "PUT", path, "user_root", JSON.stringify({ token: providerToken }), ionReferer); + assert.equal(saved.response.status, 200); + assert.equal(saved.body.configured, true); + assert.equal(saved.body.verification, "verified"); + assert.equal(saved.raw.includes(providerToken), false); + + const terrainEndpoint = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/1/endpoint`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(terrainEndpoint.status, 200); + assert.equal((await terrainEndpoint.json()).credentialMode, "gateway"); + const buildingsEndpoint = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(buildingsEndpoint.status, 200); + const buildingsEndpointRaw = await buildingsEndpoint.text(); + const buildingsEndpointBody = JSON.parse(buildingsEndpointRaw); + assert.equal(buildingsEndpointBody.credentialMode, "gateway"); + assert.equal(buildingsEndpointRaw.includes(providerState.credentials.v1.buildings), false); + assert.equal(buildingsEndpointBody.url.endsWith("/tileset.json?v=production-shaped"), true); + + const endpointCallsBeforeCacheHit = providerState.endpointRequests.get("v1:96188"); + const cachedBuildingsEndpoint = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(cachedBuildingsEndpoint.status, 200); + assert.equal(providerState.endpointRequests.get("v1:96188"), endpointCallsBeforeCacheHit); + + const terrainTile = encodeURIComponent("https://assets.ion.cesium.com/us-east-1/asset_depot/1/CesiumWorldTerrain/v1.2/0/0/0.terrain"); + const proxiedTerrain = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${terrainTile}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(proxiedTerrain.status, 200); + assert.equal(await proxiedTerrain.text(), "terrain-bytes"); + + const buildingsRoot = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(buildingsEndpointBody.url)}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(buildingsRoot.status, 200); + assert.equal((await buildingsRoot.json()).root.content.uri, "root.b3dm"); + const buildingsChildUrl = new URL("14/31780/12975.b3dm", buildingsEndpointBody.url).toString(); + const buildingsChild = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(buildingsChildUrl)}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(buildingsChild.status, 200); + assert.equal(await buildingsChild.text(), "building-bytes-v1"); + + const nestedEndpoint = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/999/endpoint`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + const nestedEndpointRaw = await nestedEndpoint.text(); + const nestedEndpointBody = JSON.parse(nestedEndpointRaw); + assert.equal(nestedEndpoint.status, 200); + assert.equal(nestedEndpointRaw.includes(providerState.credentials.v1.nested), false); + const nestedChildUrl = new URL("nested-child.b3dm", nestedEndpointBody.url).toString(); + const nestedChild = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(nestedChildUrl)}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(nestedChild.status, 200); + assert.equal(await nestedChild.text(), "nested-building-bytes-v1"); + + const attackerToken = "browser-supplied-token-must-be-removed"; + const siblingUrl = `https://assets.ion.cesium.com/us-east-1/asset_depot/999/OtherAsset/root.b3dm?access_token=${attackerToken}`; + const sibling = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(siblingUrl)}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(sibling.status, 401); + assert.equal(providerState.siblingCredentialObserved, ""); + + const rejected = await requestAdmin(port, adminSecret, "PUT", path, "user_root", JSON.stringify({ token: "invalid-private-token-should-not-replace-valid-value" }), ionReferer); + assert.equal(rejected.response.status, 422); + assert.equal(rejected.body.error, "cesium_ion_token_verification_failed"); + + // Simulate metadata persisted by an older Gateway transport. A false failure + // must be rechecked exactly once after the v4 scope/cache transport upgrade. + await writeFile(join(cacheDir, "secrets", "cesium-ion-token.metadata.json"), JSON.stringify({ + updatedAt: "2026-07-15T00:00:00.000Z", + updatedBy: "user_root", + verification: "failed", + })); + const status = await requestAdmin(port, adminSecret, "GET", path, "user_root", "", ionReferer); + assert.equal(status.response.status, 200); + assert.equal(status.body.configured, true); + assert.equal(status.body.verification, "verified"); + assert.equal(status.raw.includes(providerToken), false); + const metadata = JSON.parse(await readFile(join(cacheDir, "secrets", "cesium-ion-token.metadata.json"), "utf8")); + assert.equal(metadata.verificationTransportVersion, 4); + + // Refresh-ahead is cache-first and single-flight: concurrent callers receive + // the still-valid cached endpoint while exactly one Ion request refreshes it. + await delay(1_100); + const callsBeforeRefreshAhead = providerState.endpointRequests.get("v1:96188"); + const refreshAheadResponses = await Promise.all(Array.from({ length: 8 }, () => fetch( + `http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`, + { headers: { "x-nodedc-ion-referer": ionReferer } }, + ))); + assert.equal(refreshAheadResponses.every((response) => response.status === 200), true); + await waitFor(() => providerState.endpointRequests.get("v1:96188") >= callsBeforeRefreshAhead + 1); + await delay(100); + assert.equal(providerState.endpointRequests.get("v1:96188"), callsBeforeRefreshAhead + 1); + + const rotated = await requestAdmin(port, adminSecret, "PUT", path, "user_root", JSON.stringify({ token: rotatedProviderToken }), ionReferer); + assert.equal(rotated.response.status, 200); + assert.equal(rotated.raw.includes(rotatedProviderToken), false); + const rotatedEndpoint = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + const rotatedEndpointRaw = await rotatedEndpoint.text(); + const rotatedEndpointBody = JSON.parse(rotatedEndpointRaw); + assert.equal(rotatedEndpoint.status, 200); + assert.equal(rotatedEndpointRaw.includes(providerState.credentials.v1.buildings), false); + assert.equal(rotatedEndpointRaw.includes(providerState.credentials.v2.buildings), false); + const rotatedChildUrl = new URL("rotated.b3dm", rotatedEndpointBody.url).toString(); + const rotatedChild = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(rotatedChildUrl)}`, { headers: { "x-nodedc-ion-referer": ionReferer } }); + assert.equal(rotatedChild.status, 200); + assert.equal(await rotatedChild.text(), "building-bytes-v2"); + + const indexRaw = await readFile(join(cacheDir, "index.json"), "utf8"); + for (const secret of [providerToken, rotatedProviderToken, ...Object.values(providerState.credentials.v1), ...Object.values(providerState.credentials.v2), attackerToken]) { + assert.equal(indexRaw.includes(secret), false); + assert.equal(childOutput.includes(secret), false); + } + + const health = await fetch(`http://127.0.0.1:${port}/healthz`); + const healthRaw = await health.text(); + assert.equal(health.ok, true); + assert.equal(healthRaw.includes(providerToken), false); + assert.equal(JSON.parse(healthRaw).ionConfigured, true); + + const stored = await readFile(join(cacheDir, "secrets", "cesium-ion-token"), "utf8"); + assert.equal(stored.trim(), rotatedProviderToken); + console.log("ok: Ion scopes, refresh single-flight, token rotation, and secret boundaries hold"); +} finally { + if (child && !child.killed) { + child.kill("SIGTERM"); + await once(child, "exit").catch(() => undefined); + } + if (egress?.listening) { + egress.close(); + await once(egress, "close"); + } + await rm(root, { recursive: true, force: true }); +} + +function createCesiumEgressMock(state, expectedProxyToken) { + return createHttpServer((request, response) => { + const requestUrl = new URL(request.url || "/", "http://127.0.0.1"); + if (requestUrl.pathname !== "/proxy/cesium/fetch") { + response.writeHead(404, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "not_found" })); + return; + } + if (request.headers["x-proxy-token"] !== expectedProxyToken || request.headers.authorization) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "proxy_authorization_required" })); + return; + } + const target = new URL(requestUrl.searchParams.get("url") || "https://invalid.example/"); + if (request.headers["x-nodedc-map-referer"] !== ionReferer) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "referer_required" })); + return; + } + if (target.hostname === "assets.ion.cesium.com" && target.pathname.startsWith("/us-east-1/asset_depot/1/CesiumWorldTerrain/v1.2/")) { + if (![state.credentials.v1.terrain, state.credentials.v2.terrain].includes(target.searchParams.get("access_token"))) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "terrain_authorization_required" })); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream" }); + response.end("terrain-bytes"); + return; + } + if (target.hostname === "assets.ion.cesium.com" && target.pathname.startsWith("/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/nested/")) { + const generation = target.searchParams.get("access_token") === state.credentials.v2.nested ? "v2" + : target.searchParams.get("access_token") === state.credentials.v1.nested ? "v1" : ""; + if (!generation) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "nested_authorization_required" })); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream" }); + response.end(`nested-building-bytes-${generation}`); + return; + } + if (target.hostname === "assets.ion.cesium.com" && target.pathname.startsWith("/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/")) { + const generation = target.searchParams.get("access_token") === state.credentials.v2.buildings ? "v2" + : target.searchParams.get("access_token") === state.credentials.v1.buildings ? "v1" : ""; + if (!generation) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "buildings_authorization_required" })); + return; + } + if (target.pathname.endsWith("/tileset.json")) { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ asset: { version: "1.1" }, geometricError: 1, root: { geometricError: 0, content: { uri: "root.b3dm" } } })); + return; + } + response.writeHead(200, { "content-type": "application/octet-stream" }); + response.end(`building-bytes-${generation}`); + return; + } + if (target.hostname === "assets.ion.cesium.com" && target.pathname.startsWith("/us-east-1/asset_depot/999/")) { + state.siblingCredentialObserved = target.searchParams.get("access_token") || ""; + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "scope_isolation" })); + return; + } + const authorization = String(request.headers["x-nodedc-cesium-authorization"] || ""); + const generation = authorization === `Bearer ${providerToken}` ? "v1" : authorization === `Bearer ${rotatedProviderToken}` ? "v2" : ""; + if (target.hostname !== "api.cesium.com" || !generation) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "unauthorized" })); + return; + } + const assetId = target.pathname.match(/^\/v1\/assets\/(\d+)\/endpoint$/)?.[1]; + state.endpointRequests.set(`${generation}:${assetId}`, (state.endpointRequests.get(`${generation}:${assetId}`) || 0) + 1); + const body = assetId === "1" + ? { type: "TERRAIN", url: "https://assets.ion.cesium.com/us-east-1/asset_depot/1/CesiumWorldTerrain/v1.2/", accessToken: state.credentials[generation].terrain } + : assetId === "2" + ? { type: "IMAGERY", externalType: "BING", options: { url: "https://dev.virtualearth.net/REST/v1/Imagery/Metadata/Aerial", key: state.credentials[generation].imagery, mapStyle: "Aerial" } } + : assetId === "96188" + ? { type: "3DTILES", url: "https://assets.ion.cesium.com/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/tileset.json?v=production-shaped", accessToken: state.credentials[generation].buildings } + : assetId === "999" + ? { type: "3DTILES", url: "https://assets.ion.cesium.com/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/nested/tileset.json?v=nested", accessToken: state.credentials[generation].nested } + : null; + if (!body) { + response.writeHead(404, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "not_found" })); + return; + } + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(body)); + }); +} + +function createProviderState() { + const expiresAt = Math.floor(Date.now() / 1000) + 3600; + return { + credentials: { + v1: { + terrain: fakeJwt("terrain-v1", expiresAt), + imagery: "bing-private-endpoint-key-v1", + buildings: fakeJwt("buildings-v1", expiresAt), + nested: fakeJwt("nested-v1", expiresAt), + }, + v2: { + terrain: fakeJwt("terrain-v2", expiresAt), + imagery: "bing-private-endpoint-key-v2", + buildings: fakeJwt("buildings-v2", expiresAt), + nested: fakeJwt("nested-v2", expiresAt), + }, + }, + endpointRequests: new Map(), + siblingCredentialObserved: null, + }; +} + +function fakeJwt(subject, exp) { + const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); + return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ sub: subject, exp })}.test-signature`; +} + +async function requestAdmin(port, secret, method, pathname, actorId, body = "", referer = "") { + const timestamp = String(Date.now()); + const bodyHash = createHash("sha256").update(body).digest("hex"); + const message = `nodedc.map-gateway.admin.v2\n${method}\n${pathname}\n${timestamp}\n${actorId}\n${bodyHash}\n${referer}`; + const signature = createHmac("sha256", secret).update(message).digest("hex"); + const response = await fetch(`http://127.0.0.1:${port}${pathname}`, { + method, + headers: { + ...(body ? { "content-type": "application/json" } : {}), + "x-nodedc-admin-timestamp": timestamp, + "x-nodedc-admin-actor": actorId, + "x-nodedc-admin-signature": signature, + ...(referer ? { "x-nodedc-ion-referer": referer } : {}), + }, + body: body || undefined, + }); + const raw = await response.text(); + return { response, raw, body: JSON.parse(raw) }; +} + +async function freePort() { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +async function waitForGateway(port, child) { + let output = ""; + child.stderr.on("data", (chunk) => { output += String(chunk); }); + for (let attempt = 0; attempt < 60; attempt += 1) { + if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`); + if (response.ok) return; + } catch { /* service is still starting */ } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`gateway_start_timeout:${output}`); +} + +async function waitFor(predicate) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (predicate()) return; + await delay(20); + } + throw new Error("condition_timeout"); +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} diff --git a/services/map-gateway/scripts/smoke-live-cache-fallback.mjs b/services/map-gateway/scripts/smoke-live-cache-fallback.mjs new file mode 100644 index 0000000..2934263 --- /dev/null +++ b/services/map-gateway/scripts/smoke-live-cache-fallback.mjs @@ -0,0 +1,177 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { createServer } from "node:http"; +import { createServer as createNetServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; + +const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-live-cache-smoke-")); +const upstreamPort = await freePort(); +const gatewayPort = await freePort(); +let upstream; +let gateway; +let upstreamState = "live-1"; +let gzipRevision = 1; +let upstreamRequests = 0; + +try { + upstream = createServer((request, response) => { + upstreamRequests += 1; + if (request.url?.startsWith("/gzip/")) { + const compressed = gzipSync(JSON.stringify({ revision: gzipRevision, payload: "x".repeat(8_192) })); + response.writeHead(200, { + "content-type": "application/json", + "content-encoding": "gzip", + "content-length": String(compressed.byteLength), + "cache-control": "max-age=3600", + }); + response.end(compressed); + return; + } + if (upstreamState === "unavailable") { + response.writeHead(503, { "content-type": "text/plain" }); + response.end("unavailable"); + return; + } + response.writeHead(200, { "content-type": "text/plain", "cache-control": "max-age=3600" }); + response.end(upstreamState); + }); + upstream.listen(upstreamPort, "127.0.0.1"); + await once(upstream, "listening"); + + gateway = spawn(process.execPath, ["src/server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + NODE_ENV: "test", + PORT: String(gatewayPort), + MAP_CACHE_DIR: join(root, "cache"), + MAP_CACHE_MODE: "readwrite", + MAP_GATEWAY_ALLOW_ANONYMOUS: "true", + MAP_GATEWAY_UPSTREAM_ALLOWLIST: "127.0.0.1", + MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + await waitForGateway(gatewayPort, gateway); + + const source = `http://127.0.0.1:${upstreamPort}/terrain/layer.json`; + const first = await requestGateway(gatewayPort, source); + assert.equal(first.body, "live-1"); + assert.equal(first.response.headers.get("x-nodedc-map-cache"), "live-record"); + + upstreamState = "live-2"; + const second = await requestGateway(gatewayPort, source); + assert.equal(second.body, "live-1"); + assert.equal(["live-record", "live-cache-hit"].includes(second.response.headers.get("x-nodedc-map-cache")), true); + assert.equal(upstreamRequests, 1); + const cachedEtag = second.response.headers.get("etag"); + assert.match(cachedEtag || "", /^(?:W\/)?"[^"]+"$/); + const indexPath = join(root, "cache", "index.json"); + const indexBeforeHits = await stat(indexPath); + const indexRawBeforeHits = await readFile(indexPath, "utf8"); + + const revalidated = await requestGateway(gatewayPort, source, { headers: { "if-none-match": cachedEtag } }); + assert.equal(revalidated.response.status, 304); + assert.equal(revalidated.body, ""); + assert.equal(revalidated.response.headers.get("etag"), cachedEtag); + assert.equal(upstreamRequests, 1); + + // Browser route revisions bypass only the browser HTTP cache. They must not + // create a duplicate upstream URL or TileCache entry. + const versioned = await requestGateway(gatewayPort, `${source}?nodedc_client_revision=2`); + assert.equal(versioned.body, "live-1"); + assert.equal(versioned.response.headers.get("x-nodedc-map-cache"), "live-cache-hit"); + assert.equal(upstreamRequests, 1); + for (let attempt = 0; attempt < 4; attempt += 1) { + const repeated = await requestGateway(gatewayPort, source); + assert.equal(repeated.response.headers.get("x-nodedc-map-cache"), "live-cache-hit"); + } + const indexAfterHits = await stat(indexPath); + const indexRawAfterHits = await readFile(indexPath, "utf8"); + assert.equal(indexAfterHits.ino, indexBeforeHits.ino); + assert.equal(indexAfterHits.mtimeMs, indexBeforeHits.mtimeMs); + assert.equal(indexRawAfterHits, indexRawBeforeHits); + + // `nodedc_cache_refresh=1` is the explicit opposite of "Не + // перезаписывать cache": it goes live and replaces the stored object. + const refreshed = await requestGateway(gatewayPort, `${source}?nodedc_cache_refresh=1`); + assert.equal(refreshed.body, "live-2"); + assert.equal(refreshed.response.headers.get("x-nodedc-map-cache"), "live-refresh-record"); + assert.equal(upstreamRequests, 2); + + upstreamState = "unavailable"; + const retained = await requestGateway(gatewayPort, source); + assert.equal(retained.body, "live-2"); + assert.equal(retained.response.headers.get("x-nodedc-map-cache"), "live-cache-hit"); + assert.equal(upstreamRequests, 2); + + // The second response takes the live-network-first path with a cache entry + // already present. The Gateway must not forward the compressed upstream + // Content-Length after Node has decoded the body. + const gzipSource = `http://127.0.0.1:${upstreamPort}/gzip/layer.json`; + const gzipFirst = await requestGateway(gatewayPort, gzipSource); + assert.equal(JSON.parse(gzipFirst.body).revision, 1); + gzipRevision = 2; + const gzipSecond = await requestGateway(gatewayPort, gzipSource); + assert.equal(JSON.parse(gzipSecond.body).revision, 1); + assert.equal( + ["live-record", "live-cache-hit"].includes(gzipSecond.response.headers.get("x-nodedc-map-cache")), + true, + ); + const gzipRefresh = await requestGateway(gatewayPort, `${gzipSource}?nodedc_cache_refresh=1`); + assert.equal(JSON.parse(gzipRefresh.body).revision, 2); + // Node fetch decodes gzip while retaining the encoded upstream length. A + // streaming response cannot know the decoded length before completion, so + // it must use chunking instead of forwarding the unsafe encoded value. + assert.equal(gzipRefresh.response.headers.get("content-length"), null); + console.log("ok: TileCache hits do zero index writes, support ETag/304, and only explicit refresh replaces an object"); +} finally { + await stop(gateway); + if (upstream?.listening) { + upstream.close(); + await once(upstream, "close"); + } + await rm(root, { recursive: true, force: true }); +} + +async function requestGateway(port, source, options = {}) { + const response = await fetch(`http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(source)}`, options); + return { response, body: await response.text() }; +} + +async function freePort() { + const server = createNetServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +async function waitForGateway(port, child) { + let output = ""; + child.stderr.on("data", (chunk) => { output += String(chunk); }); + for (let attempt = 0; attempt < 80; attempt += 1) { + if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`); + if (response.ok) return; + } catch { /* service is still starting */ } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`gateway_start_timeout:${output}`); +} + +async function stop(child) { + if (child && !child.killed) { + child.kill("SIGTERM"); + await once(child, "exit").catch(() => undefined); + } +} diff --git a/services/map-gateway/scripts/smoke-no-browser-credentials.mjs b/services/map-gateway/scripts/smoke-no-browser-credentials.mjs new file mode 100644 index 0000000..0c79272 --- /dev/null +++ b/services/map-gateway/scripts/smoke-no-browser-credentials.mjs @@ -0,0 +1,117 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; + +const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-credential-smoke-")); +const cacheDir = join(root, "cache"); +const port = await freePort(); +let child; + +try { + await mkdir(join(cacheDir, "ion-endpoints"), { recursive: true }); + await writeFile(join(cacheDir, "index.json"), `${JSON.stringify({ version: 1, entries: {} })}\n`); + await writeFile(join(cacheDir, "ion-endpoints", "1.json"), `${JSON.stringify({ + assetId: "1", + type: "TERRAIN", + url: "https://assets.ion.cesium.com/asset/1/", + accessToken: "terrain-private-token", + attributions: [], + savedAt: Date.now(), + })}\n`); + await writeFile(join(cacheDir, "ion-endpoints", "2.json"), `${JSON.stringify({ + assetId: "2", + type: "IMAGERY", + externalType: "BING", + options: { + url: "https://dev.virtualearth.net/", + key: "bing-private-key", + mapStyle: "Aerial", + }, + attributions: [], + savedAt: Date.now(), + })}\n`); + const expiredBuildingsToken = fakeJwt("expired-buildings", Math.floor(Date.now() / 1000) - 60); + await writeFile(join(cacheDir, "ion-endpoints", "96188.json"), `${JSON.stringify({ + assetId: "96188", + type: "3DTILES", + url: "https://assets.ion.cesium.com/us-east-1/asset_depot/96188/OpenStreetMap/CWT/2025-04-01/tileset.json?v=expired", + accessToken: expiredBuildingsToken, + attributions: [], + savedAt: Date.now(), + })}\n`); + + child = spawn(process.execPath, ["src/server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + PORT: String(port), + MAP_CACHE_DIR: cacheDir, + MAP_GATEWAY_ALLOW_ANONYMOUS: "true", + MAP_CACHE_MODE: "readwrite", + CESIUM_ION_TOKEN: "", + CESIUM_ION_ASSET_ALLOWLIST: "1,2,96188", + MAP_GATEWAY_UPSTREAM_ALLOWLIST: "assets.ion.cesium.com,dev.virtualearth.net", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + + await waitForGateway(port, child); + for (const assetId of ["1", "2"]) { + const response = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/${assetId}/endpoint`); + assert.equal(response.status, 200); + const body = await response.text(); + assert.equal(body.includes("terrain-private-token"), false); + assert.equal(body.includes("bing-private-key"), false); + assert.equal(body.includes("accessToken"), false); + assert.equal(body.includes('"key"'), false); + const endpoint = JSON.parse(body); + assert.equal(endpoint.credentialMode, "gateway"); + } + const expired = await fetch(`http://127.0.0.1:${port}/api/map/ion/assets/96188/endpoint`); + const expiredRaw = await expired.text(); + assert.equal(expired.status, 503); + assert.equal(expiredRaw.includes(expiredBuildingsToken), false); + assert.equal(expiredRaw.includes("accessToken"), false); + console.log("ok: endpoint responses contain no credentials and a known-expired Ion JWT is never served"); +} finally { + if (child && !child.killed) { + child.kill("SIGTERM"); + await once(child, "exit").catch(() => undefined); + } + await rm(root, { recursive: true, force: true }); +} + +async function freePort() { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +async function waitForGateway(port, child) { + let output = ""; + child.stderr.on("data", (chunk) => { output += String(chunk); }); + for (let attempt = 0; attempt < 60; attempt += 1) { + if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`); + if (response.ok) return; + } catch { /* service is still starting */ } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`gateway_start_timeout:${output}`); +} + +function fakeJwt(subject, exp) { + const encode = (value) => Buffer.from(JSON.stringify(value)).toString("base64url"); + return `${encode({ alg: "HS256", typ: "JWT" })}.${encode({ sub: subject, exp })}.test-signature`; +} diff --git a/services/map-gateway/scripts/smoke-streaming-cache-fill.mjs b/services/map-gateway/scripts/smoke-streaming-cache-fill.mjs new file mode 100644 index 0000000..39e1ac2 --- /dev/null +++ b/services/map-gateway/scripts/smoke-streaming-cache-fill.mjs @@ -0,0 +1,246 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { createServer, get as httpGet } from "node:http"; +import { createServer as createNetServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-streaming-smoke-")); +const cacheDir = join(root, "cache"); +const upstreamPort = await freePort(); +const gatewayPort = await freePort(); +const distinctCount = 12; +const state = { + sharedRequests: 0, + sharedEnded: false, + abortRequests: 0, + abortEnded: false, + brokenRequests: 0, + distinctRequests: 0, + distinctResponses: [], +}; +let upstream; +let gateway; + +try { + upstream = createServer((request, response) => { + const requestUrl = new URL(request.url || "/", `http://127.0.0.1:${upstreamPort}`); + if (requestUrl.pathname === "/slow/shared") { + state.sharedRequests += 1; + response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); + response.write("first-"); + setTimeout(() => { + state.sharedEnded = true; + response.end("second"); + }, 250); + return; + } + if (requestUrl.pathname === "/slow/abort") { + state.abortRequests += 1; + response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); + response.write("abort-first-"); + setTimeout(() => { + state.abortEnded = true; + response.end("abort-second"); + }, 250); + return; + } + if (requestUrl.pathname === "/broken") { + state.brokenRequests += 1; + response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); + response.write("partial-must-not-commit"); + setTimeout(() => response.destroy(new Error("intentional_upstream_truncation")), 50); + return; + } + if (requestUrl.pathname.startsWith("/distinct/")) { + state.distinctRequests += 1; + state.distinctResponses.push({ id: requestUrl.pathname.split("/").pop(), response }); + if (state.distinctResponses.length === distinctCount) { + for (const item of state.distinctResponses) { + item.response.writeHead(200, { "content-type": "text/plain", "cache-control": "max-age=3600" }); + item.response.end(`distinct-${item.id}`); + } + } + return; + } + response.writeHead(404).end(); + }); + upstream.listen(upstreamPort, "127.0.0.1"); + await once(upstream, "listening"); + + gateway = spawn(process.execPath, ["src/server.mjs"], { + cwd: new URL("..", import.meta.url), + env: { + ...process.env, + NODE_ENV: "test", + PORT: String(gatewayPort), + MAP_CACHE_DIR: cacheDir, + MAP_CACHE_MODE: "readwrite", + MAP_GATEWAY_ALLOW_ANONYMOUS: "true", + MAP_GATEWAY_UPSTREAM_ALLOWLIST: "127.0.0.1", + MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + await waitForGateway(gatewayPort, gateway); + + const sharedSource = `http://127.0.0.1:${upstreamPort}/slow/shared`; + const sharedUrl = gatewayUrl(gatewayPort, sharedSource); + const leader = await fetch(sharedUrl); + assert.equal(leader.status, 200); + const reader = leader.body.getReader(); + const first = await reader.read(); + assert.equal(Buffer.from(first.value).toString("utf8"), "first-"); + assert.equal(state.sharedEnded, false, "first byte must reach the leader before upstream end"); + assert.equal(await indexEntryCount(cacheDir), 0, "partial body must not be published in index"); + + const followerPromise = fetch(sharedUrl).then(async (response) => ({ status: response.status, body: await response.text() })); + const leaderBody = `first-${await readRemaining(reader)}`; + const follower = await followerPromise; + assert.equal(leaderBody, "first-second"); + assert.deepEqual(follower, { status: 200, body: "first-second" }); + assert.equal(state.sharedRequests, 1, "same-key followers must share one upstream"); + assert.equal(await indexEntryCount(cacheDir), 1, "complete body must be committed"); + + const abortSource = `http://127.0.0.1:${upstreamPort}/slow/abort`; + const abortUrl = gatewayUrl(gatewayPort, abortSource); + assert.equal(await abortAfterFirstChunk(abortUrl), "abort-first-"); + const afterAbort = await fetch(abortUrl); + assert.equal(await afterAbort.text(), "abort-first-abort-second"); + assert.equal(state.abortEnded, true); + assert.equal(state.abortRequests, 1, "client abort must not cancel the server-owned cache fill"); + assert.equal(await indexEntryCount(cacheDir), 2); + + const beforeBroken = await indexEntryCount(cacheDir); + const brokenSource = `http://127.0.0.1:${upstreamPort}/broken`; + let brokenRejected = false; + try { + const broken = await fetch(gatewayUrl(gatewayPort, brokenSource)); + await broken.arrayBuffer(); + } catch { + brokenRejected = true; + } + assert.equal(brokenRejected, true); + await waitFor(async () => (await tempFiles(cacheDir)).length === 0); + assert.equal(await indexEntryCount(cacheDir), beforeBroken, "truncated upstream must not publish an index entry"); + assert.equal((await tempFiles(cacheDir)).length, 0, "truncated upstream must remove temp files"); + assert.equal((await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).status, 200, "stream failure must not stop Gateway"); + + const diagnosticsBefore = (await (await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).json()).diagnostics; + const distinctBodies = await Promise.all(Array.from({ length: distinctCount }, async (_, index) => { + const response = await fetch(gatewayUrl(gatewayPort, `http://127.0.0.1:${upstreamPort}/distinct/${index}`)); + return response.text(); + })); + assert.deepEqual(distinctBodies.sort(), Array.from({ length: distinctCount }, (_, index) => `distinct-${index}`).sort()); + await waitFor(async () => await indexEntryCount(cacheDir) === beforeBroken + distinctCount); + const diagnosticsAfter = (await (await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).json()).diagnostics; + const snapshotDelta = diagnosticsAfter.indexSnapshots - diagnosticsBefore.indexSnapshots; + assert.equal(state.distinctRequests, distinctCount); + assert.equal(snapshotDelta >= 1 && snapshotDelta <= 2, true, `expected <=2 coalesced snapshots, got ${snapshotDelta}`); + console.log("ok: cold miss streams immediately, singleflight survives abort, partial bodies stay private, and index snapshots coalesce"); +} finally { + await stop(gateway); + if (upstream?.listening) { + upstream.close(); + await once(upstream, "close"); + } + await rm(root, { recursive: true, force: true }); +} + +function gatewayUrl(port, source) { + return `http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(source)}`; +} + +async function readRemaining(reader) { + const chunks = []; + while (true) { + const next = await reader.read(); + if (next.done) break; + chunks.push(Buffer.from(next.value)); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function abortAfterFirstChunk(url) { + return new Promise((resolve, reject) => { + let resolved = false; + const request = httpGet(url, (response) => { + response.once("data", (chunk) => { + resolved = true; + resolve(String(chunk)); + response.destroy(); + }); + response.once("error", (error) => { if (!resolved) reject(error); }); + }); + request.once("error", (error) => { if (!resolved) reject(error); }); + }); +} + +async function indexEntryCount(dir) { + try { + const index = JSON.parse(await readFile(join(dir, "index.json"), "utf8")); + return Object.keys(index.entries || {}).length; + } catch (error) { + if (error?.code === "ENOENT") return 0; + throw error; + } +} + +async function tempFiles(dir) { + const found = []; + async function walk(current) { + let entries; + try { entries = await readdir(current, { withFileTypes: true }); } + catch (error) { if (error?.code === "ENOENT") return; throw error; } + for (const entry of entries) { + const path = join(current, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.name.includes(".tmp")) found.push(path); + } + } + await walk(dir); + return found; +} + +async function waitFor(predicate) { + for (let attempt = 0; attempt < 200; attempt += 1) { + if (await predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("condition_timeout"); +} + +async function freePort() { + const server = createNetServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert(address && typeof address === "object"); + const port = address.port; + server.close(); + await once(server, "close"); + return port; +} + +async function waitForGateway(port, child) { + let output = ""; + child.stderr.on("data", (chunk) => { output += String(chunk); }); + for (let attempt = 0; attempt < 80; attempt += 1) { + if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`); + try { + const response = await fetch(`http://127.0.0.1:${port}/healthz`); + if (response.ok) return; + } catch { /* service is still starting */ } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`gateway_start_timeout:${output}`); +} + +async function stop(child) { + if (child && !child.killed) { + child.kill("SIGTERM"); + await once(child, "exit").catch(() => undefined); + } +} diff --git a/services/map-gateway/src/server.mjs b/services/map-gateway/src/server.mjs index f96c507..b326caf 100644 --- a/services/map-gateway/src/server.mjs +++ b/services/map-gateway/src/server.mjs @@ -1,25 +1,54 @@ -import { createHash } from "node:crypto"; +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import { dirname, join } from "node:path"; import { Readable, Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; -const config = readConfig(); +const canonicalCesiumAssetIds = ["1", "2", "96188"]; +const cesiumVerificationTransportVersion = 4; +// The neighbouring VPN egress only serves official Cesium/Bing traffic for +// Map Gateway. Other upstreams retain their existing direct policy. +const canonicalCesiumEgressHosts = new Set([ + "api.cesium.com", + "assets.ion.cesium.com", + "dev.virtualearth.net", + "ecn.t0.tiles.virtualearth.net", + "ecn.t1.tiles.virtualearth.net", + "ecn.t2.tiles.virtualearth.net", + "ecn.t3.tiles.virtualearth.net", +]); +const config = await readConfig(); const liveCache = createCacheStore("live", config.cacheDir, true); const offlineSnapshot = config.offlineSnapshotDir ? createCacheStore("offline-snapshot", config.offlineSnapshotDir, false) : null; const inflightWrites = new Map(); +const gatewayDiagnostics = createGatewayDiagnostics(config.slowUpstreamMs); +// Provider endpoint credentials are kept only here or in the private volume. +// They are never part of an endpoint response or a browser resource URL. +const activeIonEndpoints = new Map(); +const inflightIonEndpointRefreshes = new Map(); +const credentialQueryKeys = new Set(["access_token", "accesstoken", "iontoken", "token", "key", "apikey", "api_key", "signature", "sig"]); await initialiseCacheStore(liveCache); if (offlineSnapshot) await initialiseCacheStore(offlineSnapshot); +// A Platform administrator may rotate the master token after deployment. The +// private NAS file wins over the legacy environment bootstrap on restart. +let activeCesiumIonToken = (await readStoredCesiumIonToken()) || config.bootstrapCesiumIonToken; +let ionTokenWriteQueue = Promise.resolve(); +let ionTokenGeneration = 0; const server = createServer(async (request, response) => { applyCors(request, response); if (request.method === "OPTIONS") return response.writeHead(204).end(); + let requestPath = "unknown"; try { const requestUrl = new URL(request.url || "/", `http://${request.headers.host || "127.0.0.1"}`); + requestPath = requestUrl.pathname; + if (requestUrl.pathname === "/api/map/admin/cesium-ion" && ["GET", "PUT"].includes(request.method || "")) { + return await serveCesiumIonAdmin(request, response, requestUrl.pathname); + } if (!isAllowedRequest(request)) return writeJson(response, 401, { ok: false, error: "map_gateway_auth_required" }); if (requestUrl.pathname === "/healthz" && request.method === "GET") { @@ -28,7 +57,8 @@ const server = createServer(async (request, response) => { service: "nodedc-map-gateway", cache: await cacheStats(liveCache), offlineSnapshot: offlineSnapshot ? await cacheStats(offlineSnapshot) : null, - ionConfigured: Boolean(config.cesiumIonToken), + diagnostics: gatewayDiagnostics.status(), + ionConfigured: Boolean(activeCesiumIonToken), assetAllowlist: [...config.assetAllowlist].map(Number).sort((left, right) => left - right), anonymousAccess: config.allowAnonymous, }); @@ -36,7 +66,7 @@ const server = createServer(async (request, response) => { const ionAssetMatch = requestUrl.pathname.match(/^\/api\/map\/ion\/assets\/(\d+)\/endpoint$/); if (ionAssetMatch && request.method === "GET") { - return await serveIonEndpoint(response, ionAssetMatch[1]); + return await serveIonEndpoint(request, response, ionAssetMatch[1]); } if (requestUrl.pathname === "/api/map/cache" && ["GET", "HEAD"].includes(request.method || "")) { @@ -53,9 +83,12 @@ const server = createServer(async (request, response) => { return; } const status = Number(error?.statusCode || 500); + const errorCode = safeDiagnosticCode(error?.message || "map_gateway_error"); + gatewayDiagnostics.recordGatewayFailure(errorCode); + console.warn(JSON.stringify({ event: "map_gateway_request_failed", route: requestPath, status, error: errorCode })); return writeJson(response, Number.isInteger(status) && status >= 400 && status < 600 ? status : 500, { ok: false, - error: error instanceof Error ? error.message : "map_gateway_error", + error: errorCode, }); } }); @@ -69,10 +102,13 @@ server.listen(config.port, "0.0.0.0", () => { process.on("SIGTERM", () => server.close()); process.on("SIGINT", () => server.close()); -function readConfig() { +async function readConfig() { const mode = String(process.env.MAP_CACHE_MODE || "readwrite").trim().toLowerCase(); if (!["readwrite", "readonly", "offline"].includes(mode)) throw new Error("invalid_map_cache_mode"); const allowlist = parseList(process.env.MAP_GATEWAY_UPSTREAM_ALLOWLIST || "api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net"); + const mapEgress = await readMapEgressConfig(); + const ionEndpointTtlMs = parsePositiveInt(process.env.CESIUM_ION_ENDPOINT_TTL_SECONDS, 300) * 1000; + const configuredRefreshAheadMs = parsePositiveInt(process.env.CESIUM_ION_ENDPOINT_REFRESH_AHEAD_SECONDS, 60) * 1000; return { port: parsePositiveInt(process.env.PORT, 18103), cacheDir: String(process.env.MAP_CACHE_DIR || "/var/lib/nodedc-map-cache").trim(), @@ -82,8 +118,28 @@ function readConfig() { maxObjectBytes: parsePositiveInt(process.env.MAP_CACHE_MAX_OBJECT_MB, 128) * 1024 * 1024, defaultTtlMs: parsePositiveInt(process.env.MAP_CACHE_DEFAULT_TTL_SECONDS, 604800) * 1000, upstreamTimeoutMs: parsePositiveInt(process.env.MAP_GATEWAY_UPSTREAM_TIMEOUT_SECONDS, 30) * 1000, - cesiumIonToken: String(process.env.CESIUM_ION_TOKEN || "").trim(), - assetAllowlist: new Set(parseList(process.env.CESIUM_ION_ASSET_ALLOWLIST || "1,2,96188")), + slowUpstreamMs: parsePositiveInt(process.env.MAP_GATEWAY_SLOW_UPSTREAM_SECONDS, 2) * 1000, + ionEndpointTtlMs, + ionEndpointRefreshAheadMs: Math.min(configuredRefreshAheadMs, Math.max(0, ionEndpointTtlMs - 1000)), + // Test-only loopback escape hatch for the cache policy smoke test. It is + // impossible to enable in production and never broadens the real HTTPS + // allowlist. + testAllowHttpLoopback: process.env.NODE_ENV === "test" && parseBoolean(process.env.MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK, false), + // Transitional bootstrap only. A token written through the Foundry admin + // flow is held in the private cache volume and takes precedence. + bootstrapCesiumIonToken: String(process.env.CESIUM_ION_TOKEN || "").trim(), + cesiumIonApiBase: normalizeCesiumIonApiBase(process.env.CESIUM_ION_API_BASE_URL), + cesiumIonTokenFile: String(process.env.MAP_GATEWAY_CESIUM_ION_TOKEN_FILE || join(String(process.env.MAP_CACHE_DIR || "/var/lib/nodedc-map-cache").trim(), "secrets", "cesium-ion-token")).trim(), + // The production value is read from a runner-owned, read-only file. Local + // test/dev may still inject a process-only value without creating a file. + mapGatewayAdminSecret: await readMapGatewayAdminSecret(), + mapEgress, + cesiumEgressHosts: canonicalCesiumEgressHosts, + // The Foundry Map Page contract always needs terrain (1), imagery (2), + // and the canonical buildings asset. A malformed legacy environment value + // must never turn all three into 403 before the Gateway can reach Cesium. + // Root may add further approved assets, but cannot remove these core IDs. + assetAllowlist: new Set([...canonicalCesiumAssetIds, ...parseList(process.env.CESIUM_ION_ASSET_ALLOWLIST)]), upstreamAllowlist: new Set(allowlist), legacyCacheHosts: new Set(parseList(process.env.MAP_GATEWAY_LEGACY_CACHE_HOSTS || "")), offlineProviderAllowlist: new Set(parseList(process.env.MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST || "")), @@ -93,6 +149,48 @@ function readConfig() { }; } +async function readMapEgressConfig() { + const rawUrl = String(process.env.MAP_GATEWAY_EGRESS_URL || "").trim(); + if (!rawUrl) return { url: "", proxyToken: "" }; + let url; + try { + url = new URL(rawUrl); + } catch { + throw new Error("map_gateway_egress_url_invalid"); + } + if (!['http:', 'https:'].includes(url.protocol) || !url.hostname || url.username || url.password || url.pathname !== "/" || url.search || url.hash) { + throw new Error("map_gateway_egress_url_invalid"); + } + const tokenFile = String(process.env.MAP_GATEWAY_EGRESS_PROXY_TOKEN_FILE || "").trim(); + if (!tokenFile) throw new Error("map_gateway_egress_proxy_token_file_required"); + let proxyToken; + try { + proxyToken = (await readFile(tokenFile, "utf8")).trim(); + } catch { + throw new Error("map_gateway_egress_proxy_token_file_unreadable"); + } + if (!proxyToken || proxyToken.length > 4096 || /[\u0000-\u001f\u007f\s]/.test(proxyToken)) { + throw new Error("map_gateway_egress_proxy_token_file_invalid"); + } + return { url: url.toString(), proxyToken }; +} + +async function readMapGatewayAdminSecret() { + const secretFile = String(process.env.NODEDC_MAP_GATEWAY_ADMIN_SECRET_FILE || "").trim(); + if (secretFile) { + let secret; + try { + secret = (await readFile(secretFile, "utf8")).trim(); + } catch { + throw new Error("map_gateway_admin_secret_file_unreadable"); + } + if (!/^[A-Za-z0-9_-]{48,256}$/.test(secret)) throw new Error("map_gateway_admin_secret_file_invalid"); + return secret; + } + if (process.env.NODE_ENV === "production") throw new Error("map_gateway_admin_secret_file_required"); + return String(process.env.NODEDC_MAP_GATEWAY_ADMIN_SECRET || "").trim(); +} + function parseList(value) { return String(value || "").split(",").map((part) => part.trim().toLowerCase()).filter(Boolean); } @@ -107,6 +205,43 @@ function parseBoolean(value, fallback) { return ["1", "true", "yes", "on"].includes(String(value).trim().toLowerCase()); } +function normalizeCesiumIonApiBase(value) { + const configured = String(value || "https://api.cesium.com/").trim(); + let base; + try { base = new URL(configured); } catch { throw new Error("invalid_cesium_ion_api_base"); } + const testHttp = process.env.NODE_ENV === "test" && base.protocol === "http:"; + if ((!testHttp && base.protocol !== "https:") || base.username || base.password || base.search || base.hash) { + throw new Error("invalid_cesium_ion_api_base"); + } + base.pathname = base.pathname.replace(/\/+$/, "") || "/"; + return base.toString(); +} + +function normalizeIonReferer(value) { + const raw = String(value || "").trim(); + if (!raw || raw.length > 2048) return ""; + let referer; + try { referer = new URL(raw); } catch { return ""; } + if (!['http:', 'https:'].includes(referer.protocol) || referer.username || referer.password) return ""; + referer.search = ""; + referer.hash = ""; + return referer.toString(); +} + +function requestIonReferer(request) { + // This is an internal Foundry-to-Gateway transport header. It is normalized + // before reaching Cesium and, for admin calls, included in the HMAC payload. + return normalizeIonReferer(request.headers['x-nodedc-ion-referer']); +} + +function ionRefererHeaders(ionReferer) { + return ionReferer ? { referer: ionReferer } : {}; +} + +function cesiumIonAssetEndpointUrl(assetId) { + return new URL(`v1/assets/${encodeURIComponent(assetId)}/endpoint`, config.cesiumIonApiBase).toString(); +} + function applyCors(request, response) { const origin = String(request.headers.origin || ""); if (!origin || (!config.corsOrigins.has("*") && !config.corsOrigins.has(origin))) return; @@ -116,63 +251,301 @@ function applyCors(request, response) { response.setHeader("vary", "Origin"); } +async function serveCesiumIonAdmin(request, response, pathname) { + const rawBody = request.method === "PUT" ? await readSmallBody(request) : ""; + const { actorId, ionReferer } = verifyAdminRequest(request, pathname, rawBody); + if (request.method === "GET") { + const metadata = await readCesiumIonMetadata(); + // A Bing asset endpoint request starts a billable provider session. The + // normal GET path must therefore report the persisted verification state, + // rather than re-validating every time an administrator opens the modal. + // Artifacts created before this check existed have no state: validate that + // one existing private value exactly once and then remember the safe result. + let verification = metadata?.verification || (activeCesiumIonToken ? null : "not-configured"); + // Transport v4 adds production-shaped file endpoint scopes and safe + // cache-first refresh. Recheck a persisted failure once so a false v3 + // result cannot survive the artifact upgrade forever. + const mustUpgradeLegacyFailure = metadata?.verification === "failed" + && metadata.verificationTransportVersion !== cesiumVerificationTransportVersion; + if ((!verification || mustUpgradeLegacyFailure) && activeCesiumIonToken) { + verification = (await verifyCesiumIonToken(activeCesiumIonToken, ionReferer)).verification; + if (metadata) await persistCesiumIonVerification(metadata, verification); + } + return writeJson(response, 200, { + ok: true, + configured: Boolean(activeCesiumIonToken), + updatedAt: metadata?.updatedAt || null, + updatedBy: metadata?.updatedBy || null, + verification: verification || "failed", + }); + } + + let input; + try { + input = JSON.parse(rawBody); + } catch { + throw gatewayError("invalid_cesium_ion_token", 400); + } + const token = typeof input?.token === "string" ? input.token.trim() : ""; + if (token.length < 16 || token.length > 4096 || /[\u0000-\u001f\u007f\s]/.test(token)) { + throw gatewayError("invalid_cesium_ion_token", 400); + } + const verification = await verifyCesiumIonToken(token, ionReferer); + if (verification.verification !== "verified") throw gatewayError("cesium_ion_token_verification_failed", 422); + const metadata = await persistCesiumIonToken(token, actorId, verification); + return writeJson(response, 200, { + ok: true, + configured: true, + updatedAt: metadata.updatedAt, + updatedBy: metadata.updatedBy, + verification: verification.verification, + }); +} + +async function readSmallBody(request) { + const chunks = []; + let size = 0; + for await (const chunk of request) { + size += chunk.length; + if (size > 8 * 1024) throw gatewayError("map_gateway_admin_payload_too_large", 413); + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +function verifyAdminRequest(request, pathname, body) { + if (!config.mapGatewayAdminSecret) throw gatewayError("map_gateway_admin_not_configured", 503); + const timestamp = String(request.headers["x-nodedc-admin-timestamp"] || ""); + const actorId = String(request.headers["x-nodedc-admin-actor"] || "").trim(); + const suppliedSignature = String(request.headers["x-nodedc-admin-signature"] || "").trim().toLowerCase(); + const ionReferer = normalizeIonReferer(request.headers["x-nodedc-ion-referer"]); + const timestampMs = Number(timestamp); + if (!Number.isInteger(timestampMs) || Math.abs(Date.now() - timestampMs) > 60_000 || !/^[A-Za-z0-9._:@-]{1,256}$/.test(actorId) || !/^[a-f0-9]{64}$/.test(suppliedSignature)) { + throw gatewayError("map_gateway_admin_unauthorized", 401); + } + const bodyHash = createHash("sha256").update(body).digest("hex"); + const message = `nodedc.map-gateway.admin.v2\n${request.method}\n${pathname}\n${timestamp}\n${actorId}\n${bodyHash}\n${ionReferer}`; + const expectedSignature = createHmac("sha256", config.mapGatewayAdminSecret).update(message).digest("hex"); + const expected = Buffer.from(expectedSignature, "hex"); + const supplied = Buffer.from(suppliedSignature, "hex"); + if (expected.length !== supplied.length || !timingSafeEqual(expected, supplied)) throw gatewayError("map_gateway_admin_unauthorized", 401); + return { actorId, ionReferer }; +} + +function cesiumIonMetadataFile() { + return `${config.cesiumIonTokenFile}.metadata.json`; +} + +async function readStoredCesiumIonToken() { + try { + return String(await readFile(config.cesiumIonTokenFile, "utf8")).trim(); + } catch (error) { + if (error?.code !== "ENOENT") console.warn("Cesium Ion token file ignored: unreadable"); + return ""; + } +} + +async function readCesiumIonMetadata() { + try { + const metadata = JSON.parse(await readFile(cesiumIonMetadataFile(), "utf8")); + const updatedAt = typeof metadata?.updatedAt === "string" ? metadata.updatedAt : null; + const updatedBy = typeof metadata?.updatedBy === "string" ? metadata.updatedBy : null; + const verification = metadata?.verification === "verified" || metadata?.verification === "failed" ? metadata.verification : null; + const verificationTransportVersion = Number.isInteger(metadata?.verificationTransportVersion) + ? metadata.verificationTransportVersion + : 0; + return updatedAt && updatedBy ? { updatedAt, updatedBy, verification, verificationTransportVersion } : null; + } catch (error) { + if (error?.code !== "ENOENT") console.warn("Cesium Ion token metadata ignored: invalid file"); + return null; + } +} + +async function verifyCesiumIonToken(token, ionReferer = "") { + // Settings must validate the complete Foundry Map Page contract. Accepting + // a token that reads terrain and imagery but not canonical 3D Buildings + // would defer a predictable provider 403 to the browser. + const results = await Promise.all(canonicalCesiumAssetIds.map(async (assetId) => { + try { + await fetchCesiumIonEndpoint(assetId, token, ionReferer); + return true; + } catch { + return false; + } + })); + return { verification: results.every(Boolean) ? "verified" : "failed" }; +} + +async function fetchCesiumIonEndpoint(assetId, token, ionReferer = "") { + const endpoint = await fetchWithTimeout(cesiumIonAssetEndpointUrl(assetId), { + headers: { authorization: `Bearer ${token}`, ...ionRefererHeaders(ionReferer) }, + }, { operation: "ion-endpoint", assetId }); + if (!endpoint.ok) throw gatewayError("cesium_ion_endpoint_unavailable", endpoint.status || 502); + const body = await endpoint.json(); + const next = body.type === "IMAGERY" && body.externalType === "BING" + ? { + assetId, + type: body.type, + externalType: "BING", + options: { + url: validateUpstream(String(body.options?.url || "")).toString(), + key: String(body.options?.key || ""), + mapStyle: String(body.options?.mapStyle || "Aerial"), + }, + attributions: Array.isArray(body.attributions) ? body.attributions : [], + savedAt: Date.now(), + } + : { + assetId, + type: body.type, + url: validateUpstream(String(body.url || "")).toString(), + accessToken: String(body.accessToken || ""), + attributions: Array.isArray(body.attributions) ? body.attributions : [], + savedAt: Date.now(), + }; + if (assetId === "1" && next.type !== "TERRAIN") throw gatewayError("cesium_ion_endpoint_invalid", 502); + if (assetId === "2" && (next.externalType !== "BING" || next.type !== "IMAGERY")) throw gatewayError("cesium_ion_endpoint_invalid", 502); + if (!(next.externalType === "BING" ? next.options?.key : next.accessToken)) throw gatewayError("cesium_ion_endpoint_invalid", 502); + const credentialExpiresAt = ionEndpointCredentialExpiry(next); + if (credentialExpiresAt !== null && credentialExpiresAt <= Date.now()) throw gatewayError("cesium_ion_endpoint_expired", 502); + return next; +} + +function persistCesiumIonToken(token, actorId, verification) { + const task = ionTokenWriteQueue.then(async () => { + const secretDir = dirname(config.cesiumIonTokenFile); + await mkdir(secretDir, { recursive: true, mode: 0o700 }); + await chmod(secretDir, 0o700); + const tokenTemp = `${config.cesiumIonTokenFile}.${process.pid}.${Date.now()}.tmp`; + const metadata = { + updatedAt: new Date().toISOString(), + updatedBy: actorId, + verification: verification?.verification === "verified" ? "verified" : "failed", + verificationTransportVersion: cesiumVerificationTransportVersion, + }; + await writeFile(tokenTemp, `${token}\n`, { encoding: "utf8", mode: 0o600 }); + await rename(tokenTemp, config.cesiumIonTokenFile); + await chmod(config.cesiumIonTokenFile, 0o600); + await writeCesiumIonMetadata(metadata); + // Asset-scoped endpoint credentials are derived from the master token. + // Discard the prior private endpoint cache on rotation so a fresh map load + // cannot keep using credentials minted under the old provider token. + ionTokenGeneration += 1; + activeCesiumIonToken = token; + activeIonEndpoints.clear(); + inflightIonEndpointRefreshes.clear(); + await Promise.all([...config.assetAllowlist].map((assetId) => rm(ionEndpointPath(assetId), { force: true }))); + return metadata; + }); + ionTokenWriteQueue = task.then(() => undefined, () => undefined); + return task; +} + +function persistCesiumIonVerification(expectedMetadata, verification) { + const task = ionTokenWriteQueue.then(async () => { + const current = await readCesiumIonMetadata(); + // Do not let an old GET validation race a newer token rotation. + if (!current || current.updatedAt !== expectedMetadata.updatedAt || current.updatedBy !== expectedMetadata.updatedBy) return; + await writeCesiumIonMetadata({ ...current, verification, verificationTransportVersion: cesiumVerificationTransportVersion }); + }); + ionTokenWriteQueue = task.then(() => undefined, () => undefined); + return task; +} + +async function writeCesiumIonMetadata(metadata) { + const metadataPath = cesiumIonMetadataFile(); + const metadataTemp = `${metadataPath}.${process.pid}.${Date.now()}.tmp`; + await writeFile(metadataTemp, `${JSON.stringify(metadata)}\n`, { encoding: "utf8", mode: 0o600 }); + await rename(metadataTemp, metadataPath); + await chmod(metadataPath, 0o600); +} + function isAllowedRequest(request) { return config.allowAnonymous || Boolean(request.headers[config.trustedSubjectHeader]); } -async function serveIonEndpoint(response, assetId) { +async function serveIonEndpoint(request, response, assetId) { if (!config.assetAllowlist.has(assetId)) return writeJson(response, 403, { ok: false, error: "cesium_asset_not_allowed" }); + await ionTokenWriteQueue; const cached = await readIonEndpointCache(assetId); + const cachedState = cached ? ionEndpointCacheState(cached) : null; if (config.mode === "offline") { if (!cached) return writeJson(response, 504, { ok: false, error: "cesium_ion_offline_endpoint_miss" }); - if (!isOfflineProviderAllowed(cached.url)) return writeJson(response, 409, { ok: false, error: "map_provider_offline_not_permitted" }); - return writeJson(response, 200, { ok: true, ...cached, cache: "offline-endpoint-hit" }); + if (!cachedState.usable) return writeJson(response, 504, { ok: false, error: "cesium_ion_offline_endpoint_expired" }); + if (!isOfflineProviderAllowed(ionEndpointUrl(cached))) return writeJson(response, 409, { ok: false, error: "map_provider_offline_not_permitted" }); + rememberIonEndpoint(cached); + return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "offline-endpoint-hit" }); } - if (!config.cesiumIonToken) { - if (cached) return writeJson(response, 200, { ok: true, ...cached, cache: "cached-endpoint-no-master-token" }); + if (!activeCesiumIonToken) { + if (cached && cachedState.usable) { + rememberIonEndpoint(cached); + return writeJson(response, 200, { ok: true, ...publicIonEndpoint(cached), cache: "cached-endpoint-no-master-token" }); + } return writeJson(response, 503, { ok: false, error: "cesium_ion_not_configured" }); } - try { - const endpoint = await fetchWithTimeout(`https://api.cesium.com/v1/assets/${assetId}/endpoint`, { - headers: { authorization: `Bearer ${config.cesiumIonToken}` }, + const ionReferer = requestIonReferer(request); + if (cached && cachedState.usable) { + rememberIonEndpoint(cached); + if (cachedState.needsRefresh) scheduleIonEndpointRefresh(assetId, ionReferer); + return writeJson(response, 200, { + ok: true, + ...publicIonEndpoint(cached), + cache: cachedState.needsRefresh ? "ion-endpoint-refresh-ahead" : "ion-endpoint-hit", }); - if (!endpoint.ok) throw gatewayError("cesium_ion_endpoint_unavailable", endpoint.status || 502); - const body = await endpoint.json(); - const next = body.type === "IMAGERY" && body.externalType === "BING" - ? { - assetId, - type: body.type, - externalType: "BING", - options: { - url: validateUpstream(String(body.options?.url || "")).toString(), - key: String(body.options?.key || ""), - mapStyle: String(body.options?.mapStyle || "Aerial"), - }, - attributions: Array.isArray(body.attributions) ? body.attributions : [], - savedAt: Date.now(), - } - : { - assetId, - type: body.type, - url: validateUpstream(String(body.url || "")).toString(), - accessToken: String(body.accessToken || ""), - attributions: Array.isArray(body.attributions) ? body.attributions : [], - savedAt: Date.now(), - }; - if (!(next.externalType === "BING" ? next.options?.key : next.accessToken)) throw gatewayError("cesium_ion_endpoint_invalid", 502); - if (config.mode === "readwrite") await writeIonEndpointCache(next); - return writeJson(response, 200, { ok: true, ...next, cache: "ion-endpoint-online" }); - } catch (error) { - if (cached) return writeJson(response, 200, { ok: true, ...cached, cache: "stale-ion-endpoint" }); - throw error; } + + const next = await refreshIonEndpoint(assetId, ionReferer); + return writeJson(response, 200, { ok: true, ...publicIonEndpoint(next), cache: "ion-endpoint-online" }); +} + +function scheduleIonEndpointRefresh(assetId, ionReferer = "") { + const active = activeIonEndpoints.get(String(assetId)); + if (active) { + const state = ionEndpointCacheState(active); + if (state.usable && !state.needsRefresh) return; + } + void refreshIonEndpoint(assetId, ionReferer).catch((error) => { + const errorCode = safeDiagnosticCode(error?.message || "cesium_ion_endpoint_refresh_failed"); + if (errorCode === "cesium_ion_token_rotated") return; + console.warn(JSON.stringify({ event: "map_ion_endpoint_refresh_failed", assetId, error: errorCode })); + }); +} + +function refreshIonEndpoint(assetId, ionReferer = "") { + const token = activeCesiumIonToken; + if (!token) return Promise.reject(gatewayError("cesium_ion_not_configured", 503)); + const generation = ionTokenGeneration; + const existing = inflightIonEndpointRefreshes.get(assetId); + if (existing?.generation === generation) return existing.promise; + + const promise = (async () => { + const next = await fetchCesiumIonEndpoint(assetId, token, ionReferer); + // An administrator may rotate the master token while this provider request + // is in flight. Never re-introduce an endpoint credential minted under the + // previous token generation. + if (generation !== ionTokenGeneration || token !== activeCesiumIonToken) { + throw gatewayError("cesium_ion_token_rotated", 409); + } + rememberIonEndpoint(next); + if (config.mode === "readwrite") await writeIonEndpointCache(next); + return next; + })(); + const record = { generation, promise }; + inflightIonEndpointRefreshes.set(assetId, record); + promise.finally(() => { + if (inflightIonEndpointRefreshes.get(assetId) === record) inflightIonEndpointRefreshes.delete(assetId); + }).catch(() => undefined); + return promise; } async function serveCachedUpstream(request, response, requestUrl) { const rawTarget = requestUrl.searchParams.get("url"); if (!rawTarget) return writeJson(response, 400, { ok: false, error: "map_cache_url_required" }); - const target = validateUpstream(rawTarget); + let target = validateUpstream(rawTarget); + // Browser-facing route revision only. It must not affect the upstream URL + // or create a duplicate TileCache object. + target.searchParams.delete("nodedc_client_revision"); const cacheProfile = String(target.searchParams.get("nodedc_cache_profile") || "live").toLowerCase(); target.searchParams.delete("nodedc_cache_profile"); if (!new Set(["live", "offline"]).has(cacheProfile)) return writeJson(response, 400, { ok: false, error: "map_cache_profile_invalid" }); @@ -184,6 +557,7 @@ async function serveCachedUpstream(request, response, requestUrl) { const isLegacyCacheHost = config.legacyCacheHosts.has(target.hostname.toLowerCase()); const forceRefresh = target.searchParams.get("nodedc_cache_refresh") === "1"; target.searchParams.delete("nodedc_cache_refresh"); + const ionReferer = requestIonReferer(request); const cacheKey = createHash("sha256").update(canonicalCacheUrl(target)).digest("hex"); // "Live" without cache is still routed through Gateway — it is never a @@ -191,40 +565,91 @@ async function serveCachedUpstream(request, response, requestUrl) { // persistent store. The immutable offline profile intentionally has no such // bypass mode. if (cacheMode === "passthrough" && cacheProfile === "live") { - return proxyUncachedRequest(request, response, target, "live-pass-through"); + gatewayDiagnostics.recordCache("passthrough"); + target = await injectGatewayCredentials(target, ionReferer); + return proxyUncachedRequest(request, response, target, "live-pass-through", ionReferer); } const cached = await getCachedEntry(store, cacheKey); const fresh = cached && cached.expiresAt > Date.now(); - if (cached && (!forceRefresh || !store.mutable) && (fresh || !store.mutable || config.mode !== "readwrite")) { - return serveCachedFile(request, response, store, cached, fresh ? `${store.name}-hit` : `${store.name}-stale`); + // A refresh can have a previously published object at this key while its + // replacement is still streaming into the private temp file. Treat every + // request that arrives during that window as a follower: otherwise a normal + // cache-first request could race the atomic rename and receive the old + // object immediately after the refresh leader received the new one. + const activeFill = inflightWrites.get(`${store.name}:${cacheKey}`); + if (activeFill) { + try { + const entry = await activeFill.commit; + const state = cached ? "live-cache-hit" : (forceRefresh ? "live-refresh-record" : "live-record"); + return serveCachedFile(request, response, store, entry, state); + } catch (error) { + if (cached) { + gatewayDiagnostics.recordCache("staleFallback"); + return serveCachedFile(request, response, store, cached, "live-stale-upstream-error"); + } + throw error; + } } // A migrated Engine cache is a local sandbox snapshot, not an instruction to // fetch arbitrary historic providers when an object is missing. The profile // is selected by the visual adapter; the browser can never turn it into a // new upstream fetch by changing a setting. - if (cacheProfile === "offline") return writeJson(response, 504, { ok: false, error: "map_offline_snapshot_miss" }); - if (isLegacyCacheHost) return writeJson(response, 504, { ok: false, error: "map_legacy_cache_miss" }); - - if (config.mode === "offline") { - if (!isOfflineProviderAllowed(target.toString())) return writeJson(response, 409, { ok: false, error: "map_provider_offline_not_permitted" }); - return writeJson(response, 504, { ok: false, error: "map_cache_offline_miss" }); + if (cacheProfile === "offline" || config.mode === "offline") { + if (cached) return serveCachedFile(request, response, store, cached, fresh ? `${store.name}-offline-hit` : `${store.name}-offline-stale`); + return writeJson(response, 504, { ok: false, error: cacheProfile === "offline" ? "map_offline_snapshot_miss" : "map_cache_offline_miss" }); + } + if (isLegacyCacheHost) { + if (cached) return serveCachedFile(request, response, store, cached, fresh ? "legacy-cache-hit" : "legacy-cache-stale"); + return writeJson(response, 504, { ok: false, error: "map_legacy_cache_miss" }); } + // The product setting "Не перезаписывать cache" is cache-first by + // definition: a collected tile is served from the shared persistent volume + // immediately, while only a miss travels through the official live + // provider. `nodedc_cache_refresh=1` is the explicit opt-in to replace a + // previously collected object. + if (cached && !forceRefresh) { + gatewayDiagnostics.recordCache(fresh ? "hit" : "staleHit"); + return serveCachedFile(request, response, store, cached, fresh ? "live-cache-hit" : "live-cache-stale"); + } + + // Credentials are needed only for a real upstream request. Cache lookup uses + // the credential-free canonical URL, so a warm TileCache remains independent + // from Ion token refresh and from the AMD/VPN route. + target = await injectGatewayCredentials(target, ionReferer); + if (config.mode === "readonly") { - return proxyUncachedRequest(request, response, target, "pass-through-readonly"); + gatewayDiagnostics.recordCache("readonlyPassThrough"); + return proxyUncachedRequest(request, response, target, "pass-through-readonly", ionReferer); } if (String(request.headers.range || "").trim()) { - return proxyRangeRequest(request, response, target, cached, store); + gatewayDiagnostics.recordCache(cached ? "rangeRefresh" : "rangeMiss"); + return proxyRangeRequest(request, response, target, cached, store, ionReferer); } try { - const entry = await fetchAndCache(target, cacheKey, store); - return serveCachedFile(request, response, store, entry, forceRefresh ? "live-refresh-record" : "live-record"); + gatewayDiagnostics.recordCache(forceRefresh && cached ? "refresh" : "miss"); + // Keep the await inside this try block: setup failures (notably an explicit + // refresh whose provider is temporarily unavailable) must reach the stale + // fallback below instead of bypassing it as an unobserved returned promise. + return await streamAndCacheMiss( + request, + response, + target, + cacheKey, + store, + forceRefresh ? "live-refresh-record" : "live-record", + ionReferer, + ); } catch (error) { - if (cached) return serveCachedFile(request, response, store, cached, "live-stale-upstream-error"); + if (isCacheCapacityError(error)) return proxyUncachedRequest(request, response, target, "live-pass-through-cache-full", ionReferer); + if (cached) { + gatewayDiagnostics.recordCache("staleFallback"); + return serveCachedFile(request, response, store, cached, "live-stale-upstream-error"); + } throw error; } } @@ -249,10 +674,13 @@ function validateUpstream(rawTarget) { // allowed through as HTTP: upgrade only the known Bing tile hosts before // the standard HTTPS and allowlist checks. This keeps the browser and the // Gateway on encrypted transport while making the provider contract work. - if (target.protocol === "http:" && /^(ecn\.t[0-3]\.tiles\.virtualearth\.net|ecn\.tiles\.virtualearth\.net)$/.test(hostname)) { + if (target.protocol === "http:" && /^(dev\.virtualearth\.net|ecn\.t[0-3]\.tiles\.virtualearth\.net|ecn\.tiles\.virtualearth\.net)$/.test(hostname)) { target.protocol = "https:"; } - if (target.protocol !== "https:") throw gatewayError("map_cache_https_required", 400); + const testLoopbackHttp = config.testAllowHttpLoopback + && target.protocol === "http:" + && ["127.0.0.1", "localhost", "::1"].includes(hostname); + if (target.protocol !== "https:" && !testLoopbackHttp) throw gatewayError("map_cache_https_required", 400); if (target.username || target.password || (!config.upstreamAllowlist.has(hostname) && !config.legacyCacheHosts.has(hostname))) { throw gatewayError("map_cache_upstream_not_allowed", 403); } @@ -260,7 +688,7 @@ function validateUpstream(rawTarget) { } function canonicalCacheUrl(target) { - const url = new URL(target.toString()); + const url = stripCredentialQueryParameters(target); // The old Engine snapshot distributes identical Bing tile semantics across // t0…t3. A viewer can choose a different subdomain for the same quadkey, so // use one logical host for the cache key while preserving the original URL @@ -268,39 +696,76 @@ function canonicalCacheUrl(target) { if (/^ecn\.t[0-3]\.tiles\.virtualearth\.net$/i.test(url.hostname)) { url.hostname = "ecn.tiles.virtualearth.net"; } - for (const key of [...url.searchParams.keys()]) { - if (["access_token", "accesstoken", "iontoken", "token", "key", "apikey", "api_key", "signature", "sig"].includes(key.toLowerCase())) { - url.searchParams.delete(key); - } - } url.searchParams.sort(); return url.toString(); } -async function fetchAndCache(target, cacheKey, store) { +async function streamAndCacheMiss(request, response, target, cacheKey, store, cacheState, ionReferer = "") { const inflightKey = `${store.name}:${cacheKey}`; const existing = inflightWrites.get(inflightKey); - if (existing) return existing; - const task = downloadAndCache(target, cacheKey, store); - inflightWrites.set(inflightKey, task); - // Several Cesium requests can share one in-flight download. Keep a rejection - // observer on the shared promise, then rethrow to each HTTP caller below. - // A 4xx/5xx tile response must never become an unhandled rejection that stops - // the gateway process. - task.catch(() => undefined); + if (existing) { + const entry = await existing.commit; + return serveCachedFile(request, response, store, entry, cacheState); + } + + const setup = prepareStreamingCacheFill(target, cacheKey, store, ionReferer); + const record = { setup, commit: null }; + record.commit = setup.then(({ cacheCommit }) => cacheCommit); + inflightWrites.set(inflightKey, record); + // Cache fill is intentionally server-owned once started. A browser can stop + // consuming its tee branch without cancelling the upstream/disk branch used + // by concurrent viewers and future offline hits. + record.commit.catch((error) => { + const errorCode = safeDiagnosticCode(error?.message || "map_cache_fill_failed"); + gatewayDiagnostics.recordGatewayFailure(errorCode); + console.warn(JSON.stringify({ event: "map_cache_fill_failed", error: errorCode })); + }); + record.commit.finally(() => { + if (inflightWrites.get(inflightKey) === record) inflightWrites.delete(inflightKey); + }).catch(() => undefined); + + const prepared = await setup; + if (request.method === "HEAD") { + await prepared.clientBody.cancel().catch(() => undefined); + const entry = await record.commit; + return serveCachedFile(request, response, store, entry, cacheState); + } + if (request.aborted || response.destroyed) { + await prepared.clientBody.cancel().catch(() => undefined); + return; + } + + copyUpstreamHeaders(response, prepared.headers); + response.setHeader("x-nodedc-map-cache", cacheState); + response.writeHead(prepared.status); try { - return await task; - } finally { - if (inflightWrites.get(inflightKey) === task) inflightWrites.delete(inflightKey); + await pipeline(Readable.fromWeb(prepared.clientBody), response); + } catch (error) { + if (request.aborted || response.destroyed || response.writableEnded) return; + throw error; } } -async function downloadAndCache(target, cacheKey, store) { - const upstream = await fetchWithTimeout(target, { headers: { accept: "application/json, application/octet-stream, image/*, */*;q=0.5" } }); - if (!upstream.ok || !upstream.body) throw gatewayError("map_upstream_unavailable", upstream.status || 502); +async function prepareStreamingCacheFill(target, cacheKey, store, ionReferer = "") { + const upstream = await fetchWithTimeout(target, { + headers: { accept: "application/json, application/octet-stream, image/*, */*;q=0.5", ...ionRefererHeaders(ionReferer) }, + }, { operation: "cache-fill" }); + if (!upstream.ok || !upstream.body) { + await upstream.body?.cancel().catch(() => undefined); + throw gatewayError("map_upstream_unavailable", upstream.status || 502); + } const expectedBytes = Number.parseInt(upstream.headers.get("content-length") || "", 10); - if (Number.isFinite(expectedBytes) && expectedBytes > config.maxObjectBytes) throw gatewayError("map_cache_object_too_large", 413); + if (Number.isFinite(expectedBytes) && expectedBytes > config.maxObjectBytes) { + await upstream.body.cancel().catch(() => undefined); + throw gatewayError("map_cache_object_too_large", 413); + } + const [clientBody, cacheBody] = upstream.body.tee(); + const cacheCommit = commitCacheBody(cacheBody, upstream.headers, cacheKey, store); + return { status: upstream.status, headers: upstream.headers, clientBody, cacheCommit }; +} + +async function commitCacheBody(cacheBody, upstreamHeaders, cacheKey, store) { const relativePath = join("objects", cacheKey.slice(0, 2), `${cacheKey}.bin`); const filePath = join(store.dir, relativePath); const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; @@ -313,29 +778,34 @@ async function downloadAndCache(target, cacheKey, store) { callback(null, chunk); }, }); + let reservationHeld = false; try { - await pipeline(Readable.fromWeb(upstream.body), limit, createWriteStream(tempPath, { flags: "wx" })); + await pipeline(Readable.fromWeb(cacheBody), limit, createWriteStream(tempPath, { flags: "wx" })); + const replacedBytes = Number(store.index.entries[cacheKey]?.bytes || 0); + if (!(await reserveCacheCapacity(store, bytes, replacedBytes))) throw cacheCapacityError(); + reservationHeld = true; await rename(tempPath, filePath); + const now = Date.now(); + const entry = { + key: cacheKey, + file: relativePath, + bytes, + contentType: upstreamHeaders.get("content-type") || "application/octet-stream", + etag: upstreamHeaders.get("etag") || null, + savedAt: now, + lastAccessAt: now, + expiresAt: now + responseTtl(upstreamHeaders.get("cache-control")), + }; + store.index.entries[cacheKey] = entry; + await releaseCacheCapacity(store, bytes); + reservationHeld = false; + await writeCacheIndex(store); + return entry; } catch (error) { + if (reservationHeld) await releaseCacheCapacity(store, bytes); await rm(tempPath, { force: true }); throw error; } - - const now = Date.now(); - const entry = { - key: cacheKey, - file: relativePath, - bytes, - contentType: upstream.headers.get("content-type") || "application/octet-stream", - etag: upstream.headers.get("etag") || null, - savedAt: now, - lastAccessAt: now, - expiresAt: now + responseTtl(upstream.headers.get("cache-control")), - }; - store.index.entries[cacheKey] = entry; - await evictCache(store); - await writeCacheIndex(store); - return entry; } function responseTtl(cacheControl) { @@ -344,8 +814,10 @@ function responseTtl(cacheControl) { return Math.min(Number.parseInt(match[1], 10) * 1000, config.defaultTtlMs); } -async function proxyRangeRequest(request, response, target, cached, store) { - const upstream = await fetchWithTimeout(target, { headers: { range: String(request.headers.range), accept: "*/*" } }); +async function proxyRangeRequest(request, response, target, cached, store, ionReferer = "") { + const upstream = await fetchWithTimeout(target, { + headers: { range: String(request.headers.range), accept: "*/*", ...ionRefererHeaders(ionReferer) }, + }, { operation: "range" }); if (!upstream.ok || !upstream.body) { if (cached) return serveCachedFile(request, response, store, cached, "stale-range-error"); throw gatewayError("map_upstream_range_unavailable", upstream.status || 502); @@ -356,13 +828,14 @@ async function proxyRangeRequest(request, response, target, cached, store) { await pipeline(Readable.fromWeb(upstream.body), response); } -async function proxyUncachedRequest(request, response, target, cacheState) { +async function proxyUncachedRequest(request, response, target, cacheState, ionReferer = "") { const upstream = await fetchWithTimeout(target, { headers: { accept: String(request.headers.accept || "application/json, application/octet-stream, image/*, */*;q=0.5"), ...(request.headers.range ? { range: String(request.headers.range) } : {}), + ...ionRefererHeaders(ionReferer), }, - }); + }, { operation: "passthrough" }); if (!upstream.ok || !upstream.body) throw gatewayError("map_upstream_unavailable", upstream.status || 502); copyUpstreamHeaders(response, upstream.headers); response.setHeader("x-nodedc-map-cache", cacheState); @@ -375,15 +848,19 @@ async function serveCachedFile(request, response, store, entry, state) { const filePath = join(store.dir, entry.file); const info = await stat(filePath); const range = parseRange(request.headers.range, info.size); + const etag = cacheEntityTag(entry, info.size); response.setHeader("content-type", entry.contentType); response.setHeader("accept-ranges", "bytes"); response.setHeader("cache-control", "public, max-age=60"); + response.setHeader("etag", etag); response.setHeader("x-nodedc-map-cache", state); response.setHeader("x-nodedc-map-cache-age", String(Math.max(0, Math.round((Date.now() - entry.savedAt) / 1000)))); - if (store.mutable) { - entry.lastAccessAt = Date.now(); - store.index.entries[entry.key] = entry; - void writeCacheIndex(store); + // This store has append-only/no-eviction policy. A hit therefore has no + // persistent metadata to update: rewriting the complete JSON index for every + // 20 KB tile made warm views slower than the provider itself. + if (ifNoneMatchMatches(request.headers["if-none-match"], etag)) { + response.writeHead(304); + return response.end(); } if (!range) { @@ -399,6 +876,24 @@ async function serveCachedFile(request, response, store, entry, state) { return pipeline(createReadStream(filePath, range), response); } +function cacheEntityTag(entry, size) { + const providerTag = String(entry.etag || "").trim(); + if (providerTag.length <= 1024 && /^(?:W\/)?"[^"\r\n]*"$/.test(providerTag)) return providerTag; + const key = /^[a-f0-9]{64}$/i.test(String(entry.key || "")) ? String(entry.key).slice(0, 32).toLowerCase() : "object"; + const revision = Number.isFinite(Number(entry.savedAt)) ? Math.max(0, Number(entry.savedAt)).toString(36) : "0"; + return `"nodedc-${key}-${Number(size).toString(16)}-${revision}"`; +} + +function ifNoneMatchMatches(rawHeader, etag) { + const raw = String(rawHeader || ""); + if (!raw || raw.length > 8192) return false; + const expected = etag.replace(/^W\//, ""); + return raw.split(",").some((part) => { + const candidate = part.trim(); + return candidate === "*" || candidate.replace(/^W\//, "") === expected; + }); +} + function parseRange(header, size) { if (!header) return null; const match = String(header).match(/^bytes=(\d*)-(\d*)$/); @@ -434,7 +929,13 @@ function createCacheStore(name, dir, mutable) { ionEndpointsDir: join(dir, "ion-endpoints"), indexPath: join(dir, "index.json"), index: { version: 1, entries: {} }, - pendingIndexWrite: Promise.resolve(), + indexRevision: 0, + persistedIndexRevision: 0, + indexWriteScheduled: false, + indexWriteRunning: false, + indexWriteWaiters: [], + reservedBytes: 0, + capacityQueue: Promise.resolve(), }; } @@ -477,45 +978,336 @@ async function writeIonEndpointCache(endpoint) { await rename(temp, target); } -function writeCacheIndex(store) { - if (!store.mutable) return Promise.resolve(); - store.pendingIndexWrite = store.pendingIndexWrite.then(async () => { - const tempPath = `${store.indexPath}.${process.pid}.tmp`; - await writeFile(tempPath, `${JSON.stringify(store.index)}\n`, "utf8"); - await rename(tempPath, store.indexPath); - }).catch((error) => console.warn("Map cache index write failed", error instanceof Error ? error.message : "unknown")); - return store.pendingIndexWrite; +function rememberIonEndpoint(endpoint) { + if (endpoint?.assetId === undefined) return; + const assetId = String(endpoint.assetId); + const current = activeIonEndpoints.get(assetId); + if (!current || Number(endpoint.savedAt || 0) >= Number(current.savedAt || 0)) activeIonEndpoints.set(assetId, endpoint); } -async function evictCache(store) { - let entries = Object.values(store.index.entries); - let size = entries.reduce((sum, entry) => sum + Number(entry.bytes || 0), 0); - if (size <= config.maxCacheBytes) return; - entries = entries.sort((left, right) => Number(left.lastAccessAt || 0) - Number(right.lastAccessAt || 0)); - for (const entry of entries) { - if (size <= config.maxCacheBytes) break; - await rm(join(store.dir, entry.file), { force: true }); - delete store.index.entries[entry.key]; - size -= Number(entry.bytes || 0); +function ionEndpointCacheState(endpoint, now = Date.now()) { + const savedAt = Number(endpoint?.savedAt || 0); + const credentialExpiresAt = ionEndpointCredentialExpiry(endpoint); + const usable = credentialExpiresAt === null || now < credentialExpiresAt; + const ttlRefreshAt = savedAt > 0 + ? savedAt + Math.max(0, config.ionEndpointTtlMs - config.ionEndpointRefreshAheadMs) + : 0; + const credentialRefreshAt = credentialExpiresAt === null + ? Number.POSITIVE_INFINITY + : Math.max(0, credentialExpiresAt - config.ionEndpointRefreshAheadMs); + return { + usable, + needsRefresh: !savedAt || now >= Math.min(ttlRefreshAt, credentialRefreshAt), + credentialExpiresAt, + }; +} + +function ionEndpointCredentialExpiry(endpoint) { + if (endpoint?.externalType === "BING") return null; + const token = String(endpoint?.accessToken || ""); + const parts = token.split("."); + if (parts.length !== 3 || !parts[1]) return null; + try { + const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); + const seconds = Number(payload?.exp); + return Number.isFinite(seconds) && seconds > 0 ? Math.floor(seconds * 1000) : null; + } catch { + return null; } } +function ionEndpointUrl(endpoint) { + return endpoint?.externalType === "BING" ? endpoint.options?.url : endpoint?.url; +} + +function publicIonEndpoint(endpoint) { + const shared = { + assetId: endpoint.assetId, + type: endpoint.type, + attributions: Array.isArray(endpoint.attributions) ? endpoint.attributions : [], + credentialMode: "gateway", + }; + if (endpoint.externalType === "BING") { + return { + ...shared, + externalType: "BING", + options: { + url: publicProviderUrl(endpoint.options?.url), + mapStyle: String(endpoint.options?.mapStyle || "Aerial"), + }, + }; + } + return { ...shared, url: publicProviderUrl(endpoint.url) }; +} + +function publicProviderUrl(rawUrl) { + return stripCredentialQueryParameters(validateUpstream(String(rawUrl || ""))).toString(); +} + +function stripCredentialQueryParameters(target) { + const sanitized = new URL(target.toString()); + for (const key of [...sanitized.searchParams.keys()]) { + if (credentialQueryKeys.has(key.toLowerCase())) sanitized.searchParams.delete(key); + } + return sanitized; +} + +async function injectGatewayCredentials(target, ionReferer = "") { + await ionTokenWriteQueue; + const upstream = stripCredentialQueryParameters(target); + const endpoints = await knownIonEndpoints(); + let matchedIon = endpoints + .filter((endpoint) => endpoint.externalType !== "BING" && endpoint.accessToken) + .map((endpoint) => ({ endpoint, scopeLength: endpointResourceScopeLength(upstream, endpoint.url) })) + .filter(({ scopeLength }) => scopeLength >= 0) + .sort((left, right) => right.scopeLength - left.scopeLength)[0]?.endpoint; + if (matchedIon) { + const state = ionEndpointCacheState(matchedIon); + if (!state.usable) { + const refreshed = await refreshIonEndpoint(String(matchedIon.assetId), ionReferer); + if (endpointResourceScopeLength(upstream, refreshed.url) < 0) throw gatewayError("cesium_ion_endpoint_scope_changed", 409); + matchedIon = refreshed; + } else if (state.needsRefresh) { + scheduleIonEndpointRefresh(String(matchedIon.assetId), ionReferer); + } + } + if (matchedIon) upstream.searchParams.set("access_token", matchedIon.accessToken); + + const bing = endpoints.find((endpoint) => endpoint.externalType === "BING" && endpoint.options?.key); + if (bing && isBingResource(upstream)) upstream.searchParams.set("key", bing.options.key); + return upstream; +} + +async function knownIonEndpoints() { + for (const assetId of config.assetAllowlist) { + if (activeIonEndpoints.has(assetId)) continue; + const cached = await readIonEndpointCache(assetId); + if (cached) rememberIonEndpoint(cached); + } + return [...activeIonEndpoints.values()]; +} + +function endpointResourceScopeLength(target, endpointUrl) { + if (!endpointUrl) return -1; + try { + const endpoint = new URL(endpointUrl); + if (target.origin !== endpoint.origin) return -1; + if (target.pathname === endpoint.pathname) return 1_000_000 + endpoint.pathname.length; + // Ion may return a root file such as `tileset.json`; its child URIs resolve + // beside that file, not below `tileset.json/`. The directory is trusted only + // because this endpoint and credential came from the private Ion response. + // Never broaden a root-level file into an origin-wide credential scope. + const directoryPath = new URL(".", endpoint).pathname; + if (directoryPath === "/" || !target.pathname.startsWith(directoryPath)) return -1; + return directoryPath.length; + } catch { + return -1; + } +} + +function isBingResource(target) { + return /(^|\.)virtualearth\.net$/i.test(target.hostname); +} + +function writeCacheIndex(store) { + if (!store.mutable) return Promise.resolve(); + const revision = ++store.indexRevision; + const durable = new Promise((resolve, reject) => store.indexWriteWaiters.push({ revision, resolve, reject })); + scheduleCacheIndexWrite(store); + return durable; +} + +function scheduleCacheIndexWrite(store) { + if (store.indexWriteScheduled || store.indexWriteRunning) return; + store.indexWriteScheduled = true; + // Batch a short burst of neighbouring I/O completions. A Cesium viewport can + // finish dozens of distinct tiles together; serialising the full index once + // per tile is unnecessary because one later snapshot contains every entry. + setTimeout(() => { + store.indexWriteScheduled = false; + void flushCacheIndex(store); + }, 10); +} + +async function flushCacheIndex(store) { + if (store.indexWriteRunning) return; + store.indexWriteRunning = true; + try { + while (store.persistedIndexRevision < store.indexRevision) { + const snapshotRevision = store.indexRevision; + const snapshot = `${JSON.stringify(store.index)}\n`; + const tempPath = `${store.indexPath}.${process.pid}.${snapshotRevision}.tmp`; + await writeFile(tempPath, snapshot, "utf8"); + await rename(tempPath, store.indexPath); + store.persistedIndexRevision = snapshotRevision; + gatewayDiagnostics.recordIndexSnapshot(); + settleIndexWaiters(store); + } + } catch (error) { + console.warn("Map cache index write failed", error instanceof Error ? error.message : "unknown"); + const waiters = store.indexWriteWaiters.splice(0); + for (const waiter of waiters) waiter.reject(error); + } finally { + store.indexWriteRunning = false; + if (store.indexWriteWaiters.length && store.persistedIndexRevision < store.indexRevision) scheduleCacheIndexWrite(store); + } +} + +function settleIndexWaiters(store) { + const pending = []; + for (const waiter of store.indexWriteWaiters) { + if (waiter.revision <= store.persistedIndexRevision) waiter.resolve(); + else pending.push(waiter); + } + store.indexWriteWaiters = pending; +} + +function cacheBytes(store) { + return Object.values(store.index.entries).reduce((sum, entry) => sum + Number(entry.bytes || 0), 0); +} + +function reserveCacheCapacity(store, bytes, replacedBytes = 0) { + const action = store.capacityQueue.then(() => { + const used = Math.max(0, cacheBytes(store) - replacedBytes) + store.reservedBytes; + if (used + bytes > config.maxCacheBytes) return false; + store.reservedBytes += bytes; + return true; + }); + store.capacityQueue = action.then(() => undefined, () => undefined); + return action; +} + +function releaseCacheCapacity(store, bytes) { + const action = store.capacityQueue.then(() => { + store.reservedBytes = Math.max(0, store.reservedBytes - bytes); + }); + store.capacityQueue = action.then(() => undefined, () => undefined); + return action; +} + +function cacheCapacityError() { + return gatewayError("map_cache_capacity_reached", 507); +} + +function isCacheCapacityError(error) { + return error?.message === "map_cache_capacity_reached"; +} + async function cacheStats(store) { const entries = Object.values(store.index.entries); - const bytes = entries.reduce((sum, entry) => sum + Number(entry.bytes || 0), 0); - return { name: store.name, mode: store.mutable ? config.mode : "readonly", entries: entries.length, bytes, maxBytes: store.mutable ? config.maxCacheBytes : null, persistent: true }; + const bytes = cacheBytes(store); + return { + name: store.name, + mode: store.mutable ? config.mode : "readonly", + writePolicy: store.mutable ? "append-only-no-eviction" : "readonly", + entries: entries.length, + bytes, + maxBytes: store.mutable ? config.maxCacheBytes : null, + atCapacity: store.mutable ? bytes >= config.maxCacheBytes : false, + persistent: true, + }; } -async function fetchWithTimeout(target, options = {}) { +function shouldUseCesiumEgress(target) { + if (!config.mapEgress.url) return false; + try { + return config.cesiumEgressHosts.has(new URL(String(target)).hostname.toLowerCase()); + } catch { + return false; + } +} + +function throughCesiumEgress(target, options = {}) { + const upstreamUrl = new URL(String(target)); + const proxyUrl = new URL("proxy/cesium/fetch", config.mapEgress.url); + proxyUrl.searchParams.set("url", upstreamUrl.toString()); + const headers = new Headers(options.headers || {}); + const authorization = headers.get("authorization"); + const referer = headers.get("referer"); + headers.delete("authorization"); + headers.delete("referer"); + headers.set("x-proxy-token", config.mapEgress.proxyToken); + if (authorization) headers.set("x-nodedc-cesium-authorization", authorization); + if (referer) headers.set("x-nodedc-map-referer", referer); + return { target: proxyUrl, options: { ...options, headers } }; +} + +async function fetchWithTimeout(target, options = {}, diagnostic = {}) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), config.upstreamTimeoutMs); - try { return await fetch(target, { ...options, signal: controller.signal, redirect: "follow" }); } - catch (error) { throw gatewayError(error?.name === "AbortError" ? "map_upstream_timeout" : "map_upstream_fetch_failed", 502); } + const viaEgress = shouldUseCesiumEgress(target); + const provider = safeProviderName(target); + const operation = safeProviderOperation(diagnostic.operation); + const assetId = /^\d{1,20}$/.test(String(diagnostic.assetId || "")) ? String(diagnostic.assetId) : undefined; + const request = viaEgress + ? throughCesiumEgress(target, options) + : { target, options }; + const startedAt = Date.now(); + try { + const response = await fetch(request.target, { ...request.options, signal: controller.signal, redirect: "follow" }); + const egressError = safeEgressErrorCode(response.headers.get("x-nodedc-egress-error")); + // The restricted egress emits a short code instead of a target URL or a + // secret. Preserve it through Map Gateway so the operator can distinguish + // an AMD/VPN timeout from a real Cesium provider response. + if (egressError) { + await response.body?.cancel().catch(() => undefined); + throw gatewayError(`map_egress_${egressError}`, response.status || 502); + } + const durationMs = Date.now() - startedAt; + gatewayDiagnostics.recordUpstream({ + viaEgress, + status: response.status, + durationMs, + error: "", + }); + if (response.status >= 400) { + console.warn(JSON.stringify({ + event: "map_provider_http_failure", + provider, + operation, + ...(assetId ? { assetId } : {}), + status: response.status, + viaEgress, + durationMs, + })); + } + return response; + } catch (error) { + const errorCode = error?.message === "map_egress_amd_connector_timeout" || error?.message === "map_egress_amd_upstream_tls_timeout" + ? error.message + : error?.name === "AbortError" + ? "map_upstream_timeout" + : error?.message?.startsWith?.("map_egress_") + ? error.message + : "map_upstream_fetch_failed"; + const durationMs = Date.now() - startedAt; + gatewayDiagnostics.recordUpstream({ + viaEgress, + status: Number(error?.statusCode || 502), + durationMs, + error: errorCode, + }); + console.warn(JSON.stringify({ + event: "map_provider_transport_failure", + provider, + operation, + ...(assetId ? { assetId } : {}), + error: safeDiagnosticCode(errorCode), + viaEgress, + durationMs, + })); + throw gatewayError(errorCode, Number(error?.statusCode || 502)); + } finally { clearTimeout(timeout); } } function copyUpstreamHeaders(response, headers) { - for (const name of ["content-type", "content-length", "content-range", "accept-ranges", "etag", "last-modified"]) { + // Node's fetch transparently decodes a compressed upstream body, but retains + // the upstream Content-Length header. Forwarding that encoded byte length + // alongside the decoded stream truncates JSON (notably Cesium layer.json) + // in the browser. Cached responses calculate their own exact length; live + // pass-through responses intentionally use HTTP chunking instead. + for (const name of ["content-type", "content-range", "accept-ranges", "etag", "last-modified"]) { const value = headers.get(name); if (value) response.setHeader(name, value); } @@ -531,3 +1323,80 @@ function gatewayError(message, statusCode) { error.statusCode = statusCode; return error; } + +function createGatewayDiagnostics(slowUpstreamMs) { + const metrics = { + cacheHits: 0, + cacheStaleHits: 0, + cacheMisses: 0, + cacheRefreshes: 0, + cachePassthroughs: 0, + cacheStaleFallbacks: 0, + upstreamRequests: 0, + egressRequests: 0, + upstreamFailures: 0, + slowUpstreamRequests: 0, + indexSnapshots: 0, + lastFailure: null, + lastFailureAt: null, + }; + const recordFailure = (error) => { + if (!error) return; + metrics.lastFailure = safeDiagnosticCode(error); + metrics.lastFailureAt = new Date().toISOString(); + }; + return { + recordCache(state) { + if (state === "hit") metrics.cacheHits += 1; + else if (state === "staleHit") metrics.cacheStaleHits += 1; + else if (state === "refresh") metrics.cacheRefreshes += 1; + else if (state === "passthrough" || state === "readonlyPassThrough" || state === "rangeMiss" || state === "rangeRefresh") metrics.cachePassthroughs += 1; + else if (state === "staleFallback") metrics.cacheStaleFallbacks += 1; + else metrics.cacheMisses += 1; + }, + recordUpstream({ viaEgress, status, durationMs, error }) { + metrics.upstreamRequests += 1; + if (viaEgress) metrics.egressRequests += 1; + if (durationMs >= slowUpstreamMs) metrics.slowUpstreamRequests += 1; + if (Number(status) >= 400 || error) { + metrics.upstreamFailures += 1; + recordFailure(error || `upstream_http_${status}`); + } + }, + recordGatewayFailure(error) { + recordFailure(error); + }, + recordIndexSnapshot() { + metrics.indexSnapshots += 1; + }, + status() { + return { ...metrics }; + }, + }; +} + +function safeEgressErrorCode(value) { + const code = String(value || "").trim(); + return /^amd_[a-z0-9_.:-]{1,100}$/.test(code) ? code : ""; +} + +function safeDiagnosticCode(value) { + return String(value || "map_gateway_error").replace(/[^A-Za-z0-9_.:-]/g, "_").slice(0, 120) || "map_gateway_error"; +} + +function safeProviderName(target) { + try { + const host = new URL(String(target)).hostname.toLowerCase(); + if (host === "api.cesium.com") return "cesium-ion-api"; + if (host === "assets.ion.cesium.com") return "cesium-ion-assets"; + if (/(^|\.)virtualearth\.net$/.test(host)) return "bing-imagery"; + return "allowlisted-provider"; + } catch { + return "unknown-provider"; + } +} + +function safeProviderOperation(value) { + const operation = String(value || "resource").trim().toLowerCase(); + return /^(?:resource|ion-endpoint|cache-fill|range|passthrough)$/.test(operation) ? operation : "resource"; +} diff --git a/tools/dc-amd-pair/README.txt b/tools/dc-amd-pair/README.txt new file mode 100644 index 0000000..8646f32 --- /dev/null +++ b/tools/dc-amd-pair/README.txt @@ -0,0 +1,11 @@ +DC AMD Proxy pairing + +Use only after the NAS dc-amd-proxy service is deployed and reports +state=awaiting_pair. + +Open PowerShell on the AMD machine and run: + +powershell -NoProfile -ExecutionPolicy Bypass -File .\pair-dc-amd-proxy.ps1 + +The script reads the connector credential only inside the local Docker +container, sends it once to the NAS LAN address, and does not print it. diff --git a/tools/dc-amd-pair/SHA256SUMS b/tools/dc-amd-pair/SHA256SUMS new file mode 100644 index 0000000..fbe4e67 --- /dev/null +++ b/tools/dc-amd-pair/SHA256SUMS @@ -0,0 +1,2 @@ +b42a43ad743cf72f98fcea360a605dd6634f99fc4c238f118b96968e3b848866 pair-dc-amd-proxy.ps1 +c626a016f74ddb821bfb919afa0a463589d9adbba5715ba5577db4b64cd05a29 README.txt diff --git a/tools/dc-amd-pair/pair-dc-amd-proxy.ps1 b/tools/dc-amd-pair/pair-dc-amd-proxy.ps1 new file mode 100644 index 0000000..dcda2bc --- /dev/null +++ b/tools/dc-amd-pair/pair-dc-amd-proxy.ps1 @@ -0,0 +1,71 @@ +[CmdletBinding()] +param( + [string]$NasAddress = '172.22.0.222', + [int]$Port = 8790 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Windows PowerShell 5.1 does not necessarily autoload this assembly, unlike +# newer PowerShell editions. Load it before resolving HttpClientHandler. +Add-Type -AssemblyName System.Net.Http + +if ($Port -lt 1 -or $Port -gt 65535) { + throw "Port must be between 1 and 65535: $Port" +} + +$null = [Net.IPAddress]::Parse($NasAddress) +$docker = (Get-Command docker -CommandType Application -ErrorAction Stop | Select-Object -First 1).Path +if ([string]::IsNullOrWhiteSpace($docker)) { + throw 'Docker CLI executable could not be resolved.' +} + +# The token is read inside the already-running connector installation. It is +# never written to disk by this script and never printed to the console. +$token = $null +$body = $null +$handler = $null +$client = $null +$content = $null +$response = $null +try { + $token = (& $docker exec dc-amd-connector node -e "process.stdout.write(require('fs').readFileSync('/run/dc-amd-secrets/connector-access','utf8').trim())").Trim() + if ($LASTEXITCODE -ne 0) { + throw 'Unable to read the local connector access value. Check that dc-amd-connector is running.' + } + if ($token -notmatch '^[A-Za-z0-9_-]{48,256}$') { + throw 'The local connector access value has an invalid format.' + } + + $body = '{"connectorAccessToken":"' + $token + '"}' + $handler = [System.Net.Http.HttpClientHandler]::new() + # Pair only with the NAS LAN address. Do not use a Windows or VPN proxy for + # this one-time local handoff. + $handler.UseProxy = $false + $client = [System.Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(20) + $content = [System.Net.Http.StringContent]::new($body, [System.Text.Encoding]::UTF8, 'application/json') + $response = $client.PostAsync("http://$NasAddress`:$Port/api/pair", $content).GetAwaiter().GetResult() + $responseBody = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() + + if ([int]$response.StatusCode -ne 201) { + throw "NAS pairing was rejected (HTTP $([int]$response.StatusCode))." + } + $result = $responseBody | ConvertFrom-Json + if ($result.ok -ne $true -or $result.state -ne 'paired') { + throw 'NAS pairing returned an unexpected result.' + } + + Write-Host 'AMD connector paired with NAS. No secret value was displayed.' +} +finally { + if ($response) { $response.Dispose() } + if ($content) { $content.Dispose() } + if ($client) { $client.Dispose() } + if ($handler) { $handler.Dispose() } + $body = $null + $token = $null + Remove-Variable -Name body -ErrorAction SilentlyContinue + Remove-Variable -Name token -ErrorAction SilentlyContinue +}