#!/usr/bin/env node import { createHash } from "node:crypto"; import { copyFile, mkdir, readFile, rename, stat, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, resolve } from "node:path"; const args = new Set(process.argv.slice(2)); const positional = process.argv.slice(2).filter((value) => !value.startsWith("--")); const [manifestArgument, cacheArgument] = positional; const dryRun = args.has("--dry-run"); const refresh = args.has("--refresh"); if (!manifestArgument || !cacheArgument) { console.error("Usage: npm run import:engine-cache -- /absolute/path/to/manifest.json /persistent/map-cache [--dry-run] [--refresh]"); process.exitCode = 1; } else { await importEngineCache(resolve(manifestArgument), resolve(cacheArgument), { dryRun, refresh }); } async function importEngineCache(manifestPath, cacheDir, options) { const manifest = JSON.parse(await readFile(manifestPath, "utf8")); const sourceRoot = dirname(manifestPath); const sourceEntries = Object.entries(manifest?.entries || {}); const indexPath = join(cacheDir, "index.json"); const index = await readIndex(indexPath); const reindexed = reindexLegacyEntries(index); const report = { kind: "nodedc-engine-cache-import", importedAt: new Date().toISOString(), sourceManifest: manifestPath, sourceManifestVersion: manifest?.version ?? null, mode: options.dryRun ? "dry-run" : options.refresh ? "refresh" : "no-overwrite", totalEntries: sourceEntries.length, imported: 0, alreadyPresent: 0, missingSourceFiles: 0, invalidEntries: 0, reindexed, bytesImported: 0, providers: {}, }; if (!options.dryRun) await mkdir(join(cacheDir, "objects"), { recursive: true }); for (const [rawUrl, source] of sourceEntries) { const normalized = normalizeLegacyUrl(rawUrl); const relativeSource = String(source?.path || ""); const sourcePath = resolve(sourceRoot, relativeSource); if (!normalized || !relativeSource || isAbsolute(relativeSource) || !sourcePath.startsWith(`${sourceRoot}/`)) { report.invalidEntries += 1; continue; } let sourceInfo; try { sourceInfo = await stat(sourcePath); if (!sourceInfo.isFile()) throw new Error("not_file"); } catch { report.missingSourceFiles += 1; continue; } const key = createHash("sha256").update(canonicalCacheUrl(normalized)).digest("hex"); const targetFile = join("objects", key.slice(0, 2), `${key}.bin`); const targetPath = join(cacheDir, targetFile); const existsInIndex = Boolean(index.entries[key]); const existsOnDisk = await fileExists(targetPath); if (!options.refresh && (existsInIndex || existsOnDisk)) { report.alreadyPresent += 1; continue; } const hostname = new URL(normalized).hostname.toLowerCase(); report.providers[hostname] = (report.providers[hostname] || 0) + 1; const savedAt = Number(source?.savedAt || Date.now()); const entry = { key, file: targetFile, bytes: sourceInfo.size, contentType: String(source?.contentType || "application/octet-stream"), etag: String(source?.etag || ""), savedAt, lastAccessAt: Date.now(), // Imported records are deliberately treated as fresh in the sandbox. // A refresh is always explicit through nodedc_cache_refresh=1. expiresAt: Date.now() + 3650 * 24 * 60 * 60 * 1000, source: "engine-legacy-cache", sourceUrl: normalized, }; if (!options.dryRun) { await mkdir(dirname(targetPath), { recursive: true }); await copyFile(sourcePath, targetPath); index.entries[key] = entry; } report.imported += 1; report.bytesImported += sourceInfo.size; } if (!options.dryRun) { await writeJsonAtomically(indexPath, index); const importsDir = join(cacheDir, "imports"); await mkdir(importsDir, { recursive: true }); await writeJsonAtomically(join(importsDir, "engine-legacy-cache-import.json"), report); } process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); } function normalizeLegacyUrl(rawUrl) { try { const target = new URL(rawUrl); // The Engine cache predates the HTTPS-only Gateway. The same tile hosts // support HTTPS, so normalize only the protocol; path and provider stay intact. if (target.protocol === "http:") target.protocol = "https:"; if (target.protocol !== "https:") return null; return target; } catch { return null; } } function canonicalCacheUrl(target) { const url = new URL(target.toString()); if (/^ecn\.t[0-3]\.tiles\.virtualearth\.net$/i.test(url.hostname)) { url.hostname = "ecn.tiles.virtualearth.net"; } for (const key of [...url.searchParams.keys()]) { if (["access_token", "accesstoken", "iontoken", "token", "key", "apikey", "api_key", "signature", "sig"].includes(key.toLowerCase())) { url.searchParams.delete(key); } } url.searchParams.sort(); return url.toString(); } function reindexLegacyEntries(index) { let reindexed = 0; for (const [currentKey, entry] of Object.entries(index.entries)) { if (entry?.source !== "engine-legacy-cache" || !entry?.sourceUrl) continue; let source; try { source = new URL(String(entry.sourceUrl)); } catch { continue; } const nextKey = createHash("sha256").update(canonicalCacheUrl(source)).digest("hex"); if (nextKey === currentKey || index.entries[nextKey]) continue; index.entries[nextKey] = { ...entry, key: nextKey }; delete index.entries[currentKey]; reindexed += 1; } return reindexed; } async function readIndex(indexPath) { try { const parsed = JSON.parse(await readFile(indexPath, "utf8")); if (parsed?.version === 1 && parsed.entries && typeof parsed.entries === "object") return parsed; } catch (error) { if (error?.code !== "ENOENT") throw error; } return { version: 1, entries: {} }; } async function fileExists(path) { try { return (await stat(path)).isFile(); } catch { return false; } } async function writeJsonAtomically(path, value) { const temp = `${path}.${process.pid}.tmp`; await writeFile(temp, `${JSON.stringify(value)}\n`, "utf8"); await rename(temp, path); }