feat(map): add persistent gateway and ontology package

2026-07-13 17:14:34 +03:00
parent d196d4b0c7
commit e527812826
21 changed files with 1323 additions and 9 deletions
@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+
+import { access, readFile, stat } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+
+const manifestPath = resolve(process.argv[2] || "");
+if (!process.argv[2]) {
+  console.error("Usage: npm run audit:engine-cache -- /absolute/path/to/cesium/manifest.json");
+  process.exitCode = 1;
+} else {
+  await audit(manifestPath);
+}
+
+async function audit(path) {
+  let manifest;
+  try {
+    manifest = JSON.parse(await readFile(path, "utf8"));
+  } catch (error) {
+    throw new Error(`Cannot read legacy cache manifest: ${error instanceof Error ? error.message : "unknown error"}`);
+  }
+
+  const root = dirname(path);
+  const entries = Object.entries(manifest?.entries || {});
+  const providers = new Map();
+  let bytesFromManifest = 0;
+  let presentFiles = 0;
+  let missingFiles = 0;
+  let bytesOnDisk = 0;
+
+  for (const [rawUrl, entry] of entries) {
+    let hostname = "invalid-url";
+    try { hostname = new URL(rawUrl).hostname.toLowerCase() || "invalid-url"; } catch {}
+    const provider = providers.get(hostname) || { entries: 0, bytes: 0, presentFiles: 0, missingFiles: 0 };
+    const entryBytes = Math.max(0, Number(entry?.size || 0));
+    provider.entries += 1;
+    provider.bytes += entryBytes;
+    bytesFromManifest += entryBytes;
+
+    const relativeFile = String(entry?.path || "");
+    const candidate = resolve(root, relativeFile);
+    if (!relativeFile || !candidate.startsWith(`${root}/`) || isAbsolute(relativeFile) || !(await fileExists(candidate))) {
+      provider.missingFiles += 1;
+      missingFiles += 1;
+      providers.set(hostname, provider);
+      continue;
+    }
+
+    const fileInfo = await stat(candidate);
+    presentFiles += 1;
+    provider.presentFiles += 1;
+    bytesOnDisk += fileInfo.size;
+    providers.set(hostname, provider);
+  }
+
+  const report = {
+    kind: "nodedc-legacy-cesium-cache-audit",
+    generatedAt: new Date().toISOString(),
+    manifestPath: path,
+    manifestVersion: manifest?.version ?? null,
+    entries: entries.length,
+    declaredBytes: bytesFromManifest,
+    declaredMegabytes: toMegabytes(bytesFromManifest),
+    presentFiles,
+    missingFiles,
+    bytesOnDisk,
+    megabytesOnDisk: toMegabytes(bytesOnDisk),
+    providers: [...providers.entries()]
+      .map(([hostname, value]) => ({ hostname, ...value, megabytes: toMegabytes(value.bytes) }))
+      .sort((left, right) => right.bytes - left.bytes),
+    migrationPolicy: {
+      automaticImport: false,
+      reason: "Audit only. Import requires an explicit licence/provider decision for every source host and must preserve required attribution.",
+      target: "Map Gateway persistent volume or approved object storage; never Git history or a Docker image layer."
+    }
+  };
+
+  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+}
+
+async function fileExists(path) {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function toMegabytes(bytes) {
+  return Number((bytes / 1024 / 1024).toFixed(2));
+}
@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { copyFile, mkdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+
+const args = new Set(process.argv.slice(2));
+const positional = process.argv.slice(2).filter((value) => !value.startsWith("--"));
+const [manifestArgument, cacheArgument] = positional;
+const dryRun = args.has("--dry-run");
+const refresh = args.has("--refresh");
+
+if (!manifestArgument || !cacheArgument) {
+  console.error("Usage: npm run import:engine-cache -- /absolute/path/to/manifest.json /persistent/map-cache [--dry-run] [--refresh]");
+  process.exitCode = 1;
+} else {
+  await importEngineCache(resolve(manifestArgument), resolve(cacheArgument), { dryRun, refresh });
+}
+
+async function importEngineCache(manifestPath, cacheDir, options) {
+  const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+  const sourceRoot = dirname(manifestPath);
+  const sourceEntries = Object.entries(manifest?.entries || {});
+  const indexPath = join(cacheDir, "index.json");
+  const index = await readIndex(indexPath);
+  const reindexed = reindexLegacyEntries(index);
+  const report = {
+    kind: "nodedc-engine-cache-import",
+    importedAt: new Date().toISOString(),
+    sourceManifest: manifestPath,
+    sourceManifestVersion: manifest?.version ?? null,
+    mode: options.dryRun ? "dry-run" : options.refresh ? "refresh" : "no-overwrite",
+    totalEntries: sourceEntries.length,
+    imported: 0,
+    alreadyPresent: 0,
+    missingSourceFiles: 0,
+    invalidEntries: 0,
+    reindexed,
+    bytesImported: 0,
+    providers: {},
+  };
+
+  if (!options.dryRun) await mkdir(join(cacheDir, "objects"), { recursive: true });
+
+  for (const [rawUrl, source] of sourceEntries) {
+    const normalized = normalizeLegacyUrl(rawUrl);
+    const relativeSource = String(source?.path || "");
+    const sourcePath = resolve(sourceRoot, relativeSource);
+    if (!normalized || !relativeSource || isAbsolute(relativeSource) || !sourcePath.startsWith(`${sourceRoot}/`)) {
+      report.invalidEntries += 1;
+      continue;
+    }
+
+    let sourceInfo;
+    try {
+      sourceInfo = await stat(sourcePath);
+      if (!sourceInfo.isFile()) throw new Error("not_file");
+    } catch {
+      report.missingSourceFiles += 1;
+      continue;
+    }
+
+    const key = createHash("sha256").update(canonicalCacheUrl(normalized)).digest("hex");
+    const targetFile = join("objects", key.slice(0, 2), `${key}.bin`);
+    const targetPath = join(cacheDir, targetFile);
+    const existsInIndex = Boolean(index.entries[key]);
+    const existsOnDisk = await fileExists(targetPath);
+    if (!options.refresh && (existsInIndex || existsOnDisk)) {
+      report.alreadyPresent += 1;
+      continue;
+    }
+
+    const hostname = new URL(normalized).hostname.toLowerCase();
+    report.providers[hostname] = (report.providers[hostname] || 0) + 1;
+    const savedAt = Number(source?.savedAt || Date.now());
+    const entry = {
+      key,
+      file: targetFile,
+      bytes: sourceInfo.size,
+      contentType: String(source?.contentType || "application/octet-stream"),
+      etag: String(source?.etag || ""),
+      savedAt,
+      lastAccessAt: Date.now(),
+      // Imported records are deliberately treated as fresh in the sandbox.
+      // A refresh is always explicit through nodedc_cache_refresh=1.
+      expiresAt: Date.now() + 3650 * 24 * 60 * 60 * 1000,
+      source: "engine-legacy-cache",
+      sourceUrl: normalized,
+    };
+
+    if (!options.dryRun) {
+      await mkdir(dirname(targetPath), { recursive: true });
+      await copyFile(sourcePath, targetPath);
+      index.entries[key] = entry;
+    }
+    report.imported += 1;
+    report.bytesImported += sourceInfo.size;
+  }
+
+  if (!options.dryRun) {
+    await writeJsonAtomically(indexPath, index);
+    const importsDir = join(cacheDir, "imports");
+    await mkdir(importsDir, { recursive: true });
+    await writeJsonAtomically(join(importsDir, "engine-legacy-cache-import.json"), report);
+  }
+  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+}
+
+function normalizeLegacyUrl(rawUrl) {
+  try {
+    const target = new URL(rawUrl);
+    // The Engine cache predates the HTTPS-only Gateway. The same tile hosts
+    // support HTTPS, so normalize only the protocol; path and provider stay intact.
+    if (target.protocol === "http:") target.protocol = "https:";
+    if (target.protocol !== "https:") return null;
+    return target;
+  } catch {
+    return null;
+  }
+}
+
+function canonicalCacheUrl(target) {
+  const url = new URL(target.toString());
+  if (/^ecn\.t[0-3]\.tiles\.virtualearth\.net$/i.test(url.hostname)) {
+    url.hostname = "ecn.tiles.virtualearth.net";
+  }
+  for (const key of [...url.searchParams.keys()]) {
+    if (["access_token", "accesstoken", "iontoken", "token", "key", "apikey", "api_key", "signature", "sig"].includes(key.toLowerCase())) {
+      url.searchParams.delete(key);
+    }
+  }
+  url.searchParams.sort();
+  return url.toString();
+}
+
+function reindexLegacyEntries(index) {
+  let reindexed = 0;
+  for (const [currentKey, entry] of Object.entries(index.entries)) {
+    if (entry?.source !== "engine-legacy-cache" || !entry?.sourceUrl) continue;
+    let source;
+    try { source = new URL(String(entry.sourceUrl)); } catch { continue; }
+    const nextKey = createHash("sha256").update(canonicalCacheUrl(source)).digest("hex");
+    if (nextKey === currentKey || index.entries[nextKey]) continue;
+    index.entries[nextKey] = { ...entry, key: nextKey };
+    delete index.entries[currentKey];
+    reindexed += 1;
+  }
+  return reindexed;
+}
+
+async function readIndex(indexPath) {
+  try {
+    const parsed = JSON.parse(await readFile(indexPath, "utf8"));
+    if (parsed?.version === 1 && parsed.entries && typeof parsed.entries === "object") return parsed;
+  } catch (error) {
+    if (error?.code !== "ENOENT") throw error;
+  }
+  return { version: 1, entries: {} };
+}
+
+async function fileExists(path) {
+  try {
+    return (await stat(path)).isFile();
+  } catch {
+    return false;
+  }
+}
+
+async function writeJsonAtomically(path, value) {
+  const temp = `${path}.${process.pid}.tmp`;
+  await writeFile(temp, `${JSON.stringify(value)}\n`, "utf8");
+  await rename(temp, path);
+}