import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises"; import { join, resolve, sep } from "node:path"; export const MAP_REFERENCE_SEED_SCHEMA = "nodedc.map-reference-seed/v1"; export const MAP_REFERENCE_SNAPSHOT_SCHEMA = "nodedc.map-reference.snapshot/v1"; export const MAP_REFERENCE_SEARCH_SCHEMA = "nodedc.map-reference.search/v1"; export const TRANSPORT_STATION_PROFILE_ID = "transport-stations.v1"; const FACT_ID = /^osm\.(node|way|relation)\.([1-9]\d*)$/; const CATEGORIES = new Set(["metro", "railway_station", "railway_terminal"]); const SEMANTIC_TYPES = new Set(["map.station", "map.terminal"]); const MAX_FACTS = 100_000; const MAX_CELLS_PER_REQUEST = 64; const MAX_CELL_BYTES = 8 * 1024 * 1024; const MAX_CONCURRENT_UPSTREAM_FETCHES = 2; const MIN_UPSTREAM_START_INTERVAL_MS = 750; const MAX_SEARCH_RESULTS = 32; const SEARCH_CACHE_TTL_MS = 5 * 60 * 1000; export async function createReferenceStationSource({ seedFile, cacheDir, fetchEnabled = true, overpassApiBase = "https://overpass-api.de/api/interpreter", cellDegrees = 0.25, timeoutMs = 30_000, fetchImpl = fetch, }) { const seed = validateSeed(JSON.parse(await readFile(resolve(seedFile), "utf8"))); const cellRoot = resolve(cacheDir, "reference-features", TRANSPORT_STATION_PROFILE_ID); const searchIndexFile = resolve(cacheDir, "reference-features", `${TRANSPORT_STATION_PROFILE_ID}.search.json`); await mkdir(cellRoot, { recursive: true, mode: 0o750 }); const endpoint = normalizeOverpassEndpoint(overpassApiBase); const normalizedCellDegrees = finite(cellDegrees, 0.05, 5, "reference_station_cell_degrees_invalid"); const seedCoverage = factCoverage(seed.facts, normalizedCellDegrees); const normalizedTimeoutMs = finite(timeoutMs, 1_000, 120_000, "reference_station_timeout_invalid"); const inflightCells = new Map(); const inflightSearches = new Map(); const recentSearches = new Map(); const indexedFacts = new Map(seed.facts.map((fact) => [fact.sourceId, fact])); for (const fact of await readSearchIndex(searchIndexFile)) indexedFacts.set(fact.sourceId, fact); const pendingFetches = []; let activeFetches = 0; let upstreamRequests = 0; let upstreamFailures = 0; let lastRefreshAt = null; let lastFailure = null; let lastFailureAt = null; let searchRequests = 0; let searchFailures = 0; let upstreamStartQueue = Promise.resolve(); let nextUpstreamStartAt = 0; async function snapshot({ bbox } = {}) { const normalizedBbox = bbox ? normalizeBbox(bbox) : null; let cachedFacts = []; let requestedCells = []; let complete = true; if (normalizedBbox) { requestedCells = cellsForBbox(normalizedBbox, normalizedCellDegrees); if (requestedCells.length > MAX_CELLS_PER_REQUEST) { requestedCells = []; complete = false; } else { const results = await Promise.all(requestedCells.map(async (cell) => { if (cellInsideCoverage(cell, seedCoverage)) return { facts: [] }; const cached = await readCell(cellRoot, cell.key); if (cached) { indexFacts(cached.facts); return cached; } if (!fetchEnabled) return null; return enqueueFetch(cell); })); cachedFacts = results.flatMap((result) => result?.facts ?? []); if (results.some((result) => !result)) complete = false; } } const seedFacts = normalizedBbox ? seed.facts.filter((fact) => factInside(fact, normalizedBbox)) : seed.facts; const facts = deduplicateFacts([...seedFacts, ...cachedFacts]) .filter((fact) => !normalizedBbox || factInside(fact, normalizedBbox)); const generatedAt = new Date().toISOString(); const contentDigest = sha256(JSON.stringify(facts)); const categoryCounts = Object.fromEntries([...CATEGORIES].map((category) => [ category, facts.filter((fact) => fact.attributes.category === category).length, ])); return { schemaVersion: MAP_REFERENCE_SNAPSHOT_SCHEMA, profileId: TRANSPORT_STATION_PROFILE_ID, sourceRevision: contentDigest, generatedAt, complete, contentDigest, facts, metadata: { factCount: facts.length, categoryCounts, requestedCellCount: requestedCells.length, seedRevision: seed.sourceRevision, }, }; } async function search({ query, limit = 12 } = {}) { const displayQuery = normalizeDisplaySearchQuery(query); const normalizedQuery = normalizeSearchQuery(displayQuery); const normalizedLimit = Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || 12)); const localFacts = searchIndexedFacts(indexedFacts.values(), normalizedQuery, normalizedLimit); let upstreamFacts = []; let complete = !fetchEnabled; if (fetchEnabled && normalizedQuery.length >= 3) { const cachedSearch = recentSearches.get(normalizedQuery); if (cachedSearch && Date.now() - cachedSearch.storedAt < SEARCH_CACHE_TTL_MS) { upstreamFacts = cachedSearch.facts; complete = true; } else { const inflight = inflightSearches.get(normalizedQuery); const task = inflight ?? scheduleFetch(async () => { searchRequests += 1; upstreamRequests += 1; try { const facts = await fetchSearch(endpoint, displayQuery, normalizedLimit, normalizedTimeoutMs, fetchImpl); indexFacts(facts); await writeSearchIndex(searchIndexFile, [...indexedFacts.values()]); recentSearches.set(normalizedQuery, { storedAt: Date.now(), facts }); lastRefreshAt = new Date().toISOString(); return { facts, complete: true }; } catch (error) { searchFailures += 1; upstreamFailures += 1; lastFailure = safeUpstreamFailure(error); lastFailureAt = new Date().toISOString(); return { facts: [], complete: false }; } }); if (!inflight) { inflightSearches.set(normalizedQuery, task); void task.finally(() => inflightSearches.delete(normalizedQuery)); } const result = await task; upstreamFacts = result.facts; complete = result.complete; } } const facts = searchIndexedFacts( deduplicateFacts([...localFacts, ...upstreamFacts]), normalizedQuery, normalizedLimit, ); const generatedAt = new Date().toISOString(); const contentDigest = sha256(JSON.stringify(facts)); return { schemaVersion: MAP_REFERENCE_SEARCH_SCHEMA, profileId: TRANSPORT_STATION_PROFILE_ID, sourceRevision: contentDigest, generatedAt, complete, query: displayQuery, contentDigest, facts, metadata: { factCount: facts.length, localMatchCount: localFacts.length, }, }; } function indexFacts(facts) { for (const fact of facts) indexedFacts.set(fact.sourceId, fact); } function enqueueFetch(cell) { const inflight = inflightCells.get(cell.key); if (inflight) return inflight; const task = scheduleFetch(async () => { upstreamRequests += 1; try { const document = await fetchCell(endpoint, cell, normalizedTimeoutMs, fetchImpl); await writeCell(cellRoot, cell.key, document); lastRefreshAt = document.generatedAt; return document; } catch (error) { upstreamFailures += 1; lastFailure = safeUpstreamFailure(error); lastFailureAt = new Date().toISOString(); return null; } }); inflightCells.set(cell.key, task); void task.finally(() => inflightCells.delete(cell.key)); return task; } function scheduleFetch(run) { return new Promise((resolveTask) => { pendingFetches.push({ run, resolveTask }); pumpFetchQueue(); }); } function pumpFetchQueue() { while (activeFetches < MAX_CONCURRENT_UPSTREAM_FETCHES && pendingFetches.length) { const next = pendingFetches.shift(); activeFetches += 1; void Promise.resolve() .then(waitForUpstreamStart) .then(next.run) .then(next.resolveTask) .finally(() => { activeFetches -= 1; pumpFetchQueue(); }); } } function waitForUpstreamStart() { const turn = upstreamStartQueue.then(async () => { const waitMs = Math.max(0, nextUpstreamStartAt - Date.now()); if (waitMs) await new Promise((resolveDelay) => setTimeout(resolveDelay, waitMs)); nextUpstreamStartAt = Date.now() + MIN_UPSTREAM_START_INTERVAL_MS; }); upstreamStartQueue = turn.catch(() => undefined); return turn; } async function status() { let cachedCellCount = 0; try { cachedCellCount = (await readdir(cellRoot)).filter((name) => name.endsWith(".json")).length; } catch { cachedCellCount = 0; } return { profileId: TRANSPORT_STATION_PROFILE_ID, seedRevision: seed.sourceRevision, seedFactCount: seed.facts.length, fetchEnabled, cellDegrees: normalizedCellDegrees, cachedCellCount, upstreamRequests, upstreamFailures, searchRequests, searchFailures, upstreamState: upstreamFailures > 0 && (!lastRefreshAt || Date.parse(lastFailureAt) > Date.parse(lastRefreshAt)) ? "degraded" : (lastRefreshAt ? "ready" : "idle"), activeFetches, queuedFetches: pendingFetches.length, lastRefreshAt, lastFailure, lastFailureAt, }; } return Object.freeze({ search, snapshot, status }); } function validateSeed(value) { if (!isObject(value) || value.schemaVersion !== MAP_REFERENCE_SEED_SCHEMA || value.profileId !== TRANSPORT_STATION_PROFILE_ID || !Array.isArray(value.facts) || !value.facts.length || value.facts.length > MAX_FACTS) { throw sourceError("reference_station_seed_invalid"); } const facts = value.facts.map(validateFact); if (sha256(JSON.stringify(facts)) !== value.contentDigest) { throw sourceError("reference_station_seed_digest_mismatch"); } return Object.freeze({ ...value, facts: Object.freeze(facts) }); } function validateFact(value) { if (!isObject(value) || !FACT_ID.test(String(value.sourceId || "")) || !SEMANTIC_TYPES.has(value.semanticType) || !isIso(value.observedAt) || !isIso(value.receivedAt) || value.presentationStatus !== "active" || !isObject(value.attributes) || !CATEGORIES.has(value.attributes.category)) { throw sourceError("reference_station_fact_invalid"); } const coordinates = value.geometry?.type === "Point" ? value.geometry.coordinates : null; if (!Array.isArray(coordinates) || coordinates.length !== 2 || !Number.isFinite(coordinates[0]) || coordinates[0] < -180 || coordinates[0] > 180 || !Number.isFinite(coordinates[1]) || coordinates[1] < -90 || coordinates[1] > 90) { throw sourceError("reference_station_geometry_invalid"); } const allowedAttributes = new Set(["name", "category", "network", "operator", "official_name", "local_name", "uic_ref", "wheelchair"]); if (Object.keys(value.attributes).some((key) => !allowedAttributes.has(key))) { throw sourceError("reference_station_attribute_not_allowed"); } if (value.semanticType === "map.terminal" !== (value.attributes.category === "railway_terminal")) { throw sourceError("reference_station_semantic_type_mismatch"); } return Object.freeze(structuredClone(value)); } async function fetchCell(endpoint, cell, timeoutMs, fetchImpl) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const query = `[out:json][timeout:25];(nwr["railway"="station"](${cell.south},${cell.west},${cell.north},${cell.east});nwr["railway"="halt"](${cell.south},${cell.west},${cell.north},${cell.east}););out center tags;`; const response = await fetchImpl(endpoint, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded;charset=UTF-8", "user-agent": "NODE.DC-Map-Gateway/1.0 (+https://node-dc.ru)", }, body: new URLSearchParams({ data: query }).toString(), signal: controller.signal, redirect: "error", }); if (!response.ok) throw sourceError(`reference_station_upstream_http_${response.status}`); const raw = await response.text(); if (Buffer.byteLength(raw) > MAX_CELL_BYTES) throw sourceError("reference_station_upstream_bytes_exceeded"); const payload = JSON.parse(raw); if (!Array.isArray(payload?.elements)) throw sourceError("reference_station_upstream_invalid"); const observedAt = new Date().toISOString(); const facts = payload.elements.flatMap((element) => { try { return [normalizeOverpassElement(element, observedAt)]; } catch { return []; } }); return { schemaVersion: "nodedc.map-reference-cell/v1", key: cell.key, generatedAt: observedAt, facts: deduplicateFacts(facts), }; } finally { clearTimeout(timeout); } } async function fetchSearch(endpoint, query, limit, timeoutMs, fetchImpl) { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const expression = escapeOverpassString(query); const resultLimit = Math.max(limit, Math.min(96, limit * 3)); // Exact OSM name keys are globally indexed by Overpass. Filtering the // whole planet by railway first is several orders of magnitude slower and // routinely times out; station semantics are therefore verified locally // by the same fail-closed normalizer used by viewport cells. const overpassQuery = `[out:json][timeout:25];(nwr["name"="${expression}"];nwr["name:ru"="${expression}"];);out center tags ${resultLimit};`; const response = await fetchImpl(endpoint, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded;charset=UTF-8", "user-agent": "NODE.DC-Map-Gateway/1.0 (+https://node-dc.ru)", }, body: new URLSearchParams({ data: overpassQuery }).toString(), signal: controller.signal, redirect: "error", }); if (!response.ok) throw sourceError(`reference_station_upstream_http_${response.status}`); const raw = await response.text(); if (Buffer.byteLength(raw) > MAX_CELL_BYTES) throw sourceError("reference_station_upstream_bytes_exceeded"); const payload = JSON.parse(raw); if (!Array.isArray(payload?.elements)) throw sourceError("reference_station_upstream_invalid"); const observedAt = new Date().toISOString(); return deduplicateFacts(payload.elements.flatMap((element) => { try { return [normalizeOverpassElement(element, observedAt)]; } catch { return []; } })); } finally { clearTimeout(timeout); } } function normalizeOverpassElement(value, observedAt) { if (!isObject(value) || !new Set(["node", "way", "relation"]).has(value.type) || !Number.isSafeInteger(value.id) || value.id <= 0 || !isObject(value.tags)) { throw sourceError("reference_station_upstream_element_invalid"); } if (!new Set(["station", "halt"]).has(value.tags.railway)) { throw sourceError("reference_station_upstream_not_station"); } const longitude = Number(value.lon ?? value.center?.lon); const latitude = Number(value.lat ?? value.center?.lat); if (!Number.isFinite(longitude) || longitude < -180 || longitude > 180 || !Number.isFinite(latitude) || latitude < -90 || latitude > 90) { throw sourceError("reference_station_upstream_position_invalid"); } const name = firstString(value.tags["name:ru"], value.tags.name, value.tags.official_name, value.tags.loc_name); if (!name) throw sourceError("reference_station_upstream_name_required"); const category = classifyStation(value.tags, name); return validateFact({ sourceId: `osm.${value.type}.${value.id}`, semanticType: category === "railway_terminal" ? "map.terminal" : "map.station", observedAt, receivedAt: observedAt, attributes: compact({ name, category, network: optionalString(value.tags.network), operator: optionalString(value.tags.operator), official_name: optionalString(value.tags.official_name), local_name: optionalString(value.tags.loc_name), uic_ref: optionalString(value.tags.uic_ref), wheelchair: optionalString(value.tags.wheelchair), }), geometry: { type: "Point", coordinates: [longitude, latitude] }, presentationStatus: "active", }); } function classifyStation(tags, name) { if (tags.station === "subway" || tags.subway === "yes") return "metro"; const terminalText = `${name} ${tags.loc_name || ""} ${tags.official_name || ""}`.toLocaleLowerCase("ru"); if (/(?:^|\s)(вокзал|hauptbahnhof|central station|railway station)(?:\s|$)/u.test(terminalText)) { return "railway_terminal"; } return "railway_station"; } function cellsForBbox(bbox, cellDegrees) { const cells = []; const minX = Math.floor(bbox.west / cellDegrees); const maxX = Math.ceil(bbox.east / cellDegrees) - 1; const minY = Math.floor(bbox.south / cellDegrees); const maxY = Math.ceil(bbox.north / cellDegrees) - 1; for (let x = minX; x <= maxX; x += 1) { for (let y = minY; y <= maxY; y += 1) { const west = x * cellDegrees; const south = y * cellDegrees; cells.push({ key: `${x}_${y}`, west, south, east: Math.min(180, west + cellDegrees), north: Math.min(90, south + cellDegrees), }); } } return cells; } function factCoverage(facts, cellDegrees) { const longitudes = facts.map((fact) => fact.geometry.coordinates[0]); const latitudes = facts.map((fact) => fact.geometry.coordinates[1]); return { west: Math.floor(Math.min(...longitudes) / cellDegrees) * cellDegrees, south: Math.floor(Math.min(...latitudes) / cellDegrees) * cellDegrees, east: Math.ceil(Math.max(...longitudes) / cellDegrees) * cellDegrees, north: Math.ceil(Math.max(...latitudes) / cellDegrees) * cellDegrees, }; } function cellInsideCoverage(cell, coverage) { return cell.west >= coverage.west && cell.east <= coverage.east && cell.south >= coverage.south && cell.north <= coverage.north; } function normalizeBbox(value) { const [west, south, east, north] = Array.isArray(value) ? value.map(Number) : []; if (![west, south, east, north].every(Number.isFinite) || west < -180 || west >= east || east > 180 || south < -90 || south >= north || north > 90) { throw sourceError("reference_station_bbox_invalid"); } return { west, south, east, north }; } function factInside(fact, bbox) { const [longitude, latitude] = fact.geometry.coordinates; return longitude >= bbox.west && longitude <= bbox.east && latitude >= bbox.south && latitude <= bbox.north; } function deduplicateFacts(values) { return [...new Map(values.map((fact) => [fact.sourceId, validateFact(fact)])).values()] .sort((left, right) => left.sourceId.localeCompare(right.sourceId)); } function searchIndexedFacts(values, query, limit) { return [...values] .flatMap((fact) => { const rank = bestSearchRank([ fact.sourceId, fact.attributes.name, fact.attributes.official_name, fact.attributes.local_name, fact.attributes.uic_ref, ], query); return rank === null ? [] : [{ fact, rank }]; }) .sort((left, right) => ( left.rank - right.rank || left.fact.attributes.name.localeCompare(right.fact.attributes.name, "ru") || left.fact.sourceId.localeCompare(right.fact.sourceId) )) .slice(0, limit) .map(({ fact }) => fact); } function bestSearchRank(values, query) { let best = null; for (const value of values) { const normalized = normalizeSearchValue(value); if (!normalized) continue; let rank = null; if (normalized === query) rank = 0; else if (normalized.startsWith(query)) rank = 1; else if (normalized.split(/\s+/u).some((token) => token.startsWith(query))) rank = 2; else if (normalized.includes(query)) rank = 3; if (rank !== null && (best === null || rank < best)) best = rank; } return best; } function normalizeSearchQuery(value) { const normalized = normalizeSearchValue(value); if (normalized.length < 2 || normalized.length > 96 || /[\u0000-\u001f\u007f]/u.test(normalized)) { throw sourceError("reference_station_search_query_invalid"); } return normalized; } function normalizeDisplaySearchQuery(value) { const normalized = String(value ?? "").normalize("NFKC").trim().replace(/\s+/gu, " "); if (normalized.length < 2 || normalized.length > 96 || /[\u0000-\u001f\u007f]/u.test(normalized)) { throw sourceError("reference_station_search_query_invalid"); } return normalized; } function normalizeSearchValue(value) { return String(value ?? "") .normalize("NFKC") .trim() .toLocaleLowerCase("ru") .replace(/\s+/gu, " "); } function escapeOverpassString(value) { return String(value).replace(/[\\"]/gu, "\\$&"); } async function readCell(root, key) { try { const value = JSON.parse(await readFile(withinRoot(root, `${key}.json`), "utf8")); if (value?.schemaVersion !== "nodedc.map-reference-cell/v1" || value.key !== key || !isIso(value.generatedAt) || !Array.isArray(value.facts) || value.facts.length > MAX_FACTS) return null; return { ...value, facts: value.facts.map(validateFact) }; } catch { return null; } } async function readSearchIndex(file) { try { const value = JSON.parse(await readFile(file, "utf8")); if (value?.schemaVersion !== "nodedc.map-reference-search-index/v1" || !Array.isArray(value.facts) || value.facts.length > MAX_FACTS) return []; return value.facts.map(validateFact); } catch { return []; } } async function writeSearchIndex(file, facts) { const normalizedFacts = deduplicateFacts(facts).slice(-MAX_FACTS); const document = { schemaVersion: "nodedc.map-reference-search-index/v1", generatedAt: new Date().toISOString(), facts: normalizedFacts, }; const temp = `${file}.${process.pid}.${Date.now()}.tmp`; await writeFile(temp, `${JSON.stringify(document)}\n`, { encoding: "utf8", mode: 0o640 }); await rename(temp, file); } async function writeCell(root, key, document) { const target = withinRoot(root, `${key}.json`); const temp = withinRoot(root, `${key}.${process.pid}.${Date.now()}.tmp`); await writeFile(temp, `${JSON.stringify(document)}\n`, { encoding: "utf8", mode: 0o640 }); await rename(temp, target); } function normalizeOverpassEndpoint(value) { let endpoint; try { endpoint = new URL(String(value || "")); } catch { throw sourceError("reference_station_upstream_url_invalid"); } if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || !endpoint.hostname) { throw sourceError("reference_station_upstream_url_invalid"); } return endpoint.toString(); } function withinRoot(root, child) { const base = resolve(root); const target = resolve(base, child); if (target !== base && !target.startsWith(`${base}${sep}`)) throw sourceError("reference_station_path_escape"); return target; } function finite(value, min, max, code) { const number = Number(value); if (!Number.isFinite(number) || number < min || number > max) throw sourceError(code); return number; } function firstString(...values) { return values.map(optionalString).find(Boolean); } function optionalString(value) { if (typeof value !== "string") return undefined; const normalized = value.trim(); return normalized && normalized.length <= 256 ? normalized : undefined; } function compact(value) { return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined)); } function isObject(value) { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } function isIso(value) { return typeof value === "string" && !Number.isNaN(Date.parse(value)); } function sha256(value) { return createHash("sha256").update(value).digest("hex"); } function sourceError(code) { const clientError = code === "reference_station_bbox_invalid" || code === "reference_station_search_query_invalid"; return Object.assign(new Error(code), { code, statusCode: clientError ? 400 : 500 }); } function safeUpstreamFailure(error) { const code = String(error?.message || ""); if (/^reference_station_upstream_[a-z0-9_]+$/u.test(code)) return code.slice(0, 96); if (error?.name === "AbortError") return "reference_station_upstream_timeout"; return "reference_station_upstream_unavailable"; }