feat(map): cache and search reference stations
This commit is contained in:
Codex
parent
fe274872e4
commit
951b884c4b
|
|
@ -8,7 +8,7 @@ import { fileURLToPath } from "node:url";
|
|||
|
||||
const scriptDir = dirname(fileURLToPath(import.meta.url));
|
||||
const platformRoot = resolve(scriptDir, "../..");
|
||||
const artifactDir = resolve(scriptDir, "../deploy-artifacts");
|
||||
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
|
||||
const [patchId = "platform-map-gateway-20260714-001", ...extra] = process.argv.slice(2);
|
||||
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) throw new Error("usage: build-map-gateway-artifact.mjs [patch-id]");
|
||||
|
||||
|
|
@ -27,13 +27,32 @@ try {
|
|||
await writeFile(join(stage, "manifest.env"), `id=${patchId}\ncomponent=platform\ntype=app-overlay\n`, "utf8");
|
||||
await writeFile(join(stage, "files.txt"), `${files.map(([, destination]) => destination).join("\n")}\n`, "utf8");
|
||||
await mkdir(artifactDir, { recursive: true });
|
||||
const tar = spawnSync("python3", ["-c", "import sys,tarfile\nwith tarfile.open(sys.argv[1],'w:gz',format=tarfile.PAX_FORMAT) as a:\n [a.add(n,arcname=n,recursive=True) for n in ('manifest.env','files.txt','payload')]", target], { cwd: stage, encoding: "utf8" });
|
||||
const tar = spawnSync("python3", ["-c", canonicalTarScript(), target, stage], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024 * 1024,
|
||||
});
|
||||
if (tar.status !== 0) throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
|
||||
console.log(JSON.stringify({ ok: true, patchId, artifact: target, sha256: createHash("sha256").update(await readFile(target)).digest("hex") }, null, 2));
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function canonicalTarScript() {
|
||||
return [
|
||||
"import gzip,io,pathlib,sys,tarfile",
|
||||
"root=pathlib.Path(sys.argv[2])",
|
||||
"with open(sys.argv[1],'wb') as out:",
|
||||
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||
" for top in ('manifest.env','files.txt','payload'):",
|
||||
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||
" for x in paths:",
|
||||
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
|
||||
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function copySafe(source, destination) {
|
||||
const info = await lstat(source);
|
||||
if (info.isSymbolicLink()) throw new Error(`source_symlink_rejected:${source}`);
|
||||
|
|
|
|||
|
|
@ -279,6 +279,9 @@ services:
|
|||
MAP_GATEWAY_UPSTREAM_ALLOWLIST: ${MAP_GATEWAY_UPSTREAM_ALLOWLIST:-api.cesium.com,assets.ion.cesium.com,tile.openstreetmap.org,dev.virtualearth.net,ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtualearth.net,ecn.t2.tiles.virtualearth.net,ecn.t3.tiles.virtualearth.net}
|
||||
MAP_GATEWAY_LEGACY_CACHE_HOSTS: ${MAP_GATEWAY_LEGACY_CACHE_HOSTS:-}
|
||||
MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST: ${MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST:-}
|
||||
MAP_REFERENCE_STATION_FETCH_ENABLED: ${MAP_REFERENCE_STATION_FETCH_ENABLED:-true}
|
||||
MAP_REFERENCE_STATION_CELL_DEGREES: ${MAP_REFERENCE_STATION_CELL_DEGREES:-0.5}
|
||||
MAP_REFERENCE_STATION_OVERPASS_URL: ${MAP_REFERENCE_STATION_OVERPASS_URL:-https://overpass-api.de/api/interpreter}
|
||||
volumes:
|
||||
- type: bind
|
||||
source: ${MAP_LIVE_CACHE_HOST_DIR:-/volume1/docker/nodedc-platform/map-gateway/live-tile-cache}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ RUN apk add --no-cache su-exec
|
|||
COPY package.json ./
|
||||
COPY src ./src
|
||||
COPY zone-sources ./zone-sources
|
||||
COPY reference-sources ./reference-sources
|
||||
COPY runtime-entrypoint.mjs /app/runtime-entrypoint.mjs
|
||||
|
||||
EXPOSE 18103
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@
|
|||
- защищает proxy allowlist-ом upstream hosts, HTTPS-only правилом, лимитами размера и token-free cache key;
|
||||
- ведёт лёгкий persistent cache index и health/stats без автоматического удаления уже записанных tiles.
|
||||
- отдаёт Engine отдельный provider-neutral generation геозон из проверенного профиля: текущий backend — версионированный snapshot Дептранса, будущий API-адаптер обязан материализовать тот же envelope и не меняет L2-потребителя.
|
||||
- отдаёт Foundry provider-neutral справочный snapshot транспортных станций:
|
||||
seed Москвы из проверенного legacy MMAP donor плюс bounded spatial
|
||||
cache OSM для новых viewport. L1/L2 и пользовательские Data Product в этом
|
||||
контуре не участвуют.
|
||||
|
||||
## API
|
||||
|
||||
|
|
@ -20,8 +24,44 @@
|
|||
GET /healthz
|
||||
GET /api/map/ion/assets/:assetId/endpoint
|
||||
GET|HEAD /api/map/cache?url=<encoded-upstream-url>
|
||||
GET /api/map/reference-sources/v1/profiles/transport-stations.v1/current
|
||||
GET /api/map/reference-sources/v1/profiles/transport-stations.v1/current?bbox=west,south,east,north
|
||||
GET /api/map/reference-sources/v1/profiles/transport-stations.v1/search?q=<exact-name>&limit=12
|
||||
```
|
||||
|
||||
Reference-station route возвращает `nodedc.map-reference.snapshot/v1` и только
|
||||
два онтологических типа: `map.station` и `map.terminal`. Категории v1 —
|
||||
`metro`, `railway_station`, `railway_terminal`. Проекция allowlist-ит имена,
|
||||
operator/network, `uic_ref`, wheelchair и геометрию; raw OSM tags, upstream
|
||||
endpoint и provider credentials в snapshot отсутствуют.
|
||||
|
||||
Seed проверяется на старте и покрывает Москву без сетевого запроса. Для bbox за
|
||||
пределами покрытия Gateway делит область на фиксированные spatial cells,
|
||||
выполняет не более двух OSM-запросов одновременно, разносит старты запросов
|
||||
минимум на 750 мс, дедуплицирует одинаковые
|
||||
inflight cells и атомарно пишет нормализованный cell snapshot в
|
||||
`${MAP_CACHE_DIR}/reference-features`. Warm viewport читается из этого
|
||||
persistent cache. `name:ru` имеет приоритет перед общим `name`. Неудачный
|
||||
upstream не повреждает уже сохранённые cells; `complete=false` сообщает
|
||||
вызывающей стороне о частичном результате. `/healthz` возвращает только
|
||||
безопасный код последней ошибки, её время, counters и глубину очереди — без
|
||||
upstream endpoint или payload.
|
||||
|
||||
Поиск вне уже загруженного viewport вызывается только явным Enter пользователя.
|
||||
Gateway сначала ищет в seed и persistent search index, затем при miss выполняет
|
||||
точный OSM name lookup. Upstream selector использует глобально индексированные
|
||||
`name`/`name:ru`, а найденные элементы повторно проходят fail-closed проверку
|
||||
`railway=station|halt`. Это не autocomplete и не глобальный regex scan.
|
||||
Нормализованный результат атомарно пополняет persistent search index; после
|
||||
перелёта обычный bbox-контур загружает полную spatial cell. Одинаковые запросы
|
||||
дедуплицируются и кэшируются, а `/healthz` отдельно показывает безопасные
|
||||
search counters.
|
||||
|
||||
Граница seed округляется наружу до границ этих spatial cells. Поэтому viewport
|
||||
внутри московского покрытия всегда отвечает синхронно из packaged snapshot и
|
||||
не ожидает OSM только из-за того, что содержащая его ячейка выходит за крайнюю
|
||||
координату одной из seed-точек.
|
||||
|
||||
Внутренний route `GET|HEAD /internal/zone-sources/v1/profiles/moscow-pmd-slow-zones/current` доступен сервисам в private `engine` network и намеренно не публикуется через Caddy. На старте Gateway проверяет manifest, SHA-256 обоих исходных JSON, геометрию, замкнутость колец, уникальность identity, расписание и его связи с зонами; повреждённый или неполный generation останавливает запуск. Сейчас profile упакован в image как immutable MMap snapshot. Для будущего Дептранс API меняется producer профиля (`depttrans-api-snapshot-v1`), но endpoint, stable source IDs и `nodedc.zone-source-generation/v2` остаются прежними.
|
||||
|
||||
Внутренний admin route `GET|PUT /api/map/admin/cesium-ion` не является Browser API: он доступен только из Foundry по HMAC. Ключ подписи создаётся и хранится root-owned deploy runner в `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`, а оба сервиса получают только read-only file mount. Он не является `.env`-значением, настройкой Foundry или Cesium credential. Перед записью `PUT` проверяет candidate token через Cesium asset endpoints `1` (terrain), `2` (imagery) и `96188` (3D Buildings). Не прошедшее проверку значение не заменяет рабочее. Успешный token записывается атомарно в `${MAP_CACHE_DIR}/secrets/cesium-ion-token` с mode `0600`; ответ содержит только факт конфигурации, safe verification state и audit metadata — никогда само значение.
|
||||
|
|
@ -90,6 +130,8 @@ MAP_GATEWAY_LEGACY_CACHE_HOSTS=ecn.t0.tiles.virtualearth.net,ecn.t1.tiles.virtua
|
|||
MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST=
|
||||
MAP_CACHE_DIR=/var/lib/nodedc-map-cache
|
||||
MAP_CACHE_MODE=readwrite
|
||||
MAP_REFERENCE_STATION_FETCH_ENABLED=true
|
||||
MAP_REFERENCE_STATION_CELL_DEGREES=0.5
|
||||
```
|
||||
# Cache topology
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
"test:streaming-cache-fill": "node scripts/smoke-streaming-cache-fill.mjs",
|
||||
"test:zone-source": "node --test test/zone-source-snapshot.test.mjs",
|
||||
"test:zone-source-route": "node scripts/smoke-zone-source.mjs",
|
||||
"test:reference-stations": "node --test test/reference-station-source.test.mjs",
|
||||
"audit:engine-cache": "node scripts/audit-engine-cache.mjs",
|
||||
"import:engine-cache": "node scripts/import-engine-cache.mjs"
|
||||
},
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,110 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
|
||||
const [inputPath, outputPath] = process.argv.slice(2);
|
||||
if (!inputPath || !outputPath) {
|
||||
console.error("usage: generate-reference-station-seed.mjs <input.geojson> <output.json>");
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const source = JSON.parse(await readFile(resolve(inputPath), "utf8"));
|
||||
if (source?.type !== "FeatureCollection" || !Array.isArray(source.features)) {
|
||||
throw new Error("station_seed_feature_collection_required");
|
||||
}
|
||||
|
||||
const generatedAt = isIso(source.timestamp) ? new Date(source.timestamp).toISOString() : new Date(0).toISOString();
|
||||
const facts = source.features.map((feature, index) => normalizeFeature(feature, generatedAt, index));
|
||||
facts.sort((left, right) => left.sourceId.localeCompare(right.sourceId));
|
||||
const contentDigest = createHash("sha256")
|
||||
.update(JSON.stringify(facts))
|
||||
.digest("hex");
|
||||
const counts = Object.fromEntries(["metro", "railway_station", "railway_terminal"].map((category) => [
|
||||
category,
|
||||
facts.filter((fact) => fact.attributes.category === category).length,
|
||||
]));
|
||||
const document = {
|
||||
schemaVersion: "nodedc.map-reference-seed/v1",
|
||||
profileId: "transport-stations.v1",
|
||||
sourceRevision: `seed-${contentDigest.slice(0, 16)}`,
|
||||
generatedAt,
|
||||
contentDigest,
|
||||
complete: true,
|
||||
facts,
|
||||
metadata: {
|
||||
factCount: facts.length,
|
||||
categoryCounts: counts,
|
||||
},
|
||||
};
|
||||
|
||||
await mkdir(dirname(resolve(outputPath)), { recursive: true });
|
||||
await writeFile(resolve(outputPath), `${JSON.stringify(document)}\n`, "utf8");
|
||||
console.log(JSON.stringify({ output: resolve(outputPath), ...document.metadata, contentDigest }));
|
||||
|
||||
function normalizeFeature(feature, generatedAt, index) {
|
||||
if (feature?.type !== "Feature" || feature?.geometry?.type !== "Point") {
|
||||
throw new Error(`station_seed_feature_${index}_invalid`);
|
||||
}
|
||||
const coordinates = feature.geometry.coordinates;
|
||||
if (!Array.isArray(coordinates) || coordinates.length < 2) {
|
||||
throw new Error(`station_seed_feature_${index}_coordinates_invalid`);
|
||||
}
|
||||
const longitude = Number(coordinates[0]);
|
||||
const latitude = Number(coordinates[1]);
|
||||
if (!Number.isFinite(longitude) || longitude < -180 || longitude > 180
|
||||
|| !Number.isFinite(latitude) || latitude < -90 || latitude > 90) {
|
||||
throw new Error(`station_seed_feature_${index}_coordinates_invalid`);
|
||||
}
|
||||
const properties = feature.properties && typeof feature.properties === "object" ? feature.properties : {};
|
||||
const category = normalizeCategory(properties.sort);
|
||||
const nativeId = String(feature.id || properties["@id"] || "").trim();
|
||||
const identity = nativeId.match(/^(node|way|relation)\/([1-9]\d*)$/);
|
||||
if (!identity) throw new Error(`station_seed_feature_${index}_identity_invalid`);
|
||||
const attributes = compact({
|
||||
name: firstString(properties.name, properties["name:ru"], properties.official_name, properties.loc_name),
|
||||
category,
|
||||
network: optionalString(properties.network),
|
||||
operator: optionalString(properties.operator),
|
||||
official_name: optionalString(properties.official_name),
|
||||
local_name: optionalString(properties.loc_name),
|
||||
uic_ref: optionalString(properties.uic_ref),
|
||||
wheelchair: optionalString(properties.wheelchair),
|
||||
});
|
||||
if (!attributes.name) throw new Error(`station_seed_feature_${index}_name_required`);
|
||||
return {
|
||||
sourceId: `osm.${identity[1]}.${identity[2]}`,
|
||||
semanticType: category === "railway_terminal" ? "map.terminal" : "map.station",
|
||||
observedAt: generatedAt,
|
||||
receivedAt: generatedAt,
|
||||
attributes,
|
||||
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
||||
presentationStatus: "active",
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeCategory(value) {
|
||||
if (value === "metro") return "metro";
|
||||
if (value === "terminal") return "railway_terminal";
|
||||
if (value === "rzd_station") return "railway_station";
|
||||
throw new Error("station_seed_category_invalid");
|
||||
}
|
||||
|
||||
function firstString(...values) {
|
||||
return values.map(optionalString).find(Boolean);
|
||||
}
|
||||
|
||||
function optionalString(value) {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const normalized = value.trim();
|
||||
return normalized && normalized.length <= 256 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function compact(value) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
||||
}
|
||||
|
||||
function isIso(value) {
|
||||
return typeof value === "string" && !Number.isNaN(Date.parse(value));
|
||||
}
|
||||
|
|
@ -0,0 +1,648 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
|
||||
import { join, resolve, sep } from "node:path";
|
||||
|
||||
export const MAP_REFERENCE_SEED_SCHEMA = "nodedc.map-reference-seed/v1";
|
||||
export const MAP_REFERENCE_SNAPSHOT_SCHEMA = "nodedc.map-reference.snapshot/v1";
|
||||
export const MAP_REFERENCE_SEARCH_SCHEMA = "nodedc.map-reference.search/v1";
|
||||
export const TRANSPORT_STATION_PROFILE_ID = "transport-stations.v1";
|
||||
|
||||
const FACT_ID = /^osm\.(node|way|relation)\.([1-9]\d*)$/;
|
||||
const CATEGORIES = new Set(["metro", "railway_station", "railway_terminal"]);
|
||||
const SEMANTIC_TYPES = new Set(["map.station", "map.terminal"]);
|
||||
const MAX_FACTS = 100_000;
|
||||
const MAX_CELLS_PER_REQUEST = 64;
|
||||
const MAX_CELL_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_CONCURRENT_UPSTREAM_FETCHES = 2;
|
||||
const MIN_UPSTREAM_START_INTERVAL_MS = 750;
|
||||
const MAX_SEARCH_RESULTS = 32;
|
||||
const SEARCH_CACHE_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
export async function createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
fetchEnabled = true,
|
||||
overpassApiBase = "https://overpass-api.de/api/interpreter",
|
||||
cellDegrees = 0.25,
|
||||
timeoutMs = 30_000,
|
||||
fetchImpl = fetch,
|
||||
}) {
|
||||
const seed = validateSeed(JSON.parse(await readFile(resolve(seedFile), "utf8")));
|
||||
const cellRoot = resolve(cacheDir, "reference-features", TRANSPORT_STATION_PROFILE_ID);
|
||||
const searchIndexFile = resolve(cacheDir, "reference-features", `${TRANSPORT_STATION_PROFILE_ID}.search.json`);
|
||||
await mkdir(cellRoot, { recursive: true, mode: 0o750 });
|
||||
const endpoint = normalizeOverpassEndpoint(overpassApiBase);
|
||||
const normalizedCellDegrees = finite(cellDegrees, 0.05, 5, "reference_station_cell_degrees_invalid");
|
||||
const seedCoverage = factCoverage(seed.facts, normalizedCellDegrees);
|
||||
const normalizedTimeoutMs = finite(timeoutMs, 1_000, 120_000, "reference_station_timeout_invalid");
|
||||
const inflightCells = new Map();
|
||||
const inflightSearches = new Map();
|
||||
const recentSearches = new Map();
|
||||
const indexedFacts = new Map(seed.facts.map((fact) => [fact.sourceId, fact]));
|
||||
for (const fact of await readSearchIndex(searchIndexFile)) indexedFacts.set(fact.sourceId, fact);
|
||||
const pendingFetches = [];
|
||||
let activeFetches = 0;
|
||||
let upstreamRequests = 0;
|
||||
let upstreamFailures = 0;
|
||||
let lastRefreshAt = null;
|
||||
let lastFailure = null;
|
||||
let lastFailureAt = null;
|
||||
let searchRequests = 0;
|
||||
let searchFailures = 0;
|
||||
let upstreamStartQueue = Promise.resolve();
|
||||
let nextUpstreamStartAt = 0;
|
||||
|
||||
async function snapshot({ bbox } = {}) {
|
||||
const normalizedBbox = bbox ? normalizeBbox(bbox) : null;
|
||||
let cachedFacts = [];
|
||||
let requestedCells = [];
|
||||
let complete = true;
|
||||
if (normalizedBbox) {
|
||||
requestedCells = cellsForBbox(normalizedBbox, normalizedCellDegrees);
|
||||
if (requestedCells.length > MAX_CELLS_PER_REQUEST) {
|
||||
requestedCells = [];
|
||||
complete = false;
|
||||
} else {
|
||||
const results = await Promise.all(requestedCells.map(async (cell) => {
|
||||
if (cellInsideCoverage(cell, seedCoverage)) return { facts: [] };
|
||||
const cached = await readCell(cellRoot, cell.key);
|
||||
if (cached) {
|
||||
indexFacts(cached.facts);
|
||||
return cached;
|
||||
}
|
||||
if (!fetchEnabled) return null;
|
||||
return enqueueFetch(cell);
|
||||
}));
|
||||
cachedFacts = results.flatMap((result) => result?.facts ?? []);
|
||||
if (results.some((result) => !result)) complete = false;
|
||||
}
|
||||
}
|
||||
const seedFacts = normalizedBbox ? seed.facts.filter((fact) => factInside(fact, normalizedBbox)) : seed.facts;
|
||||
const facts = deduplicateFacts([...seedFacts, ...cachedFacts])
|
||||
.filter((fact) => !normalizedBbox || factInside(fact, normalizedBbox));
|
||||
const generatedAt = new Date().toISOString();
|
||||
const contentDigest = sha256(JSON.stringify(facts));
|
||||
const categoryCounts = Object.fromEntries([...CATEGORIES].map((category) => [
|
||||
category,
|
||||
facts.filter((fact) => fact.attributes.category === category).length,
|
||||
]));
|
||||
return {
|
||||
schemaVersion: MAP_REFERENCE_SNAPSHOT_SCHEMA,
|
||||
profileId: TRANSPORT_STATION_PROFILE_ID,
|
||||
sourceRevision: contentDigest,
|
||||
generatedAt,
|
||||
complete,
|
||||
contentDigest,
|
||||
facts,
|
||||
metadata: {
|
||||
factCount: facts.length,
|
||||
categoryCounts,
|
||||
requestedCellCount: requestedCells.length,
|
||||
seedRevision: seed.sourceRevision,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function search({ query, limit = 12 } = {}) {
|
||||
const displayQuery = normalizeDisplaySearchQuery(query);
|
||||
const normalizedQuery = normalizeSearchQuery(displayQuery);
|
||||
const normalizedLimit = Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || 12));
|
||||
const localFacts = searchIndexedFacts(indexedFacts.values(), normalizedQuery, normalizedLimit);
|
||||
let upstreamFacts = [];
|
||||
let complete = !fetchEnabled;
|
||||
if (fetchEnabled && normalizedQuery.length >= 3) {
|
||||
const cachedSearch = recentSearches.get(normalizedQuery);
|
||||
if (cachedSearch && Date.now() - cachedSearch.storedAt < SEARCH_CACHE_TTL_MS) {
|
||||
upstreamFacts = cachedSearch.facts;
|
||||
complete = true;
|
||||
} else {
|
||||
const inflight = inflightSearches.get(normalizedQuery);
|
||||
const task = inflight ?? scheduleFetch(async () => {
|
||||
searchRequests += 1;
|
||||
upstreamRequests += 1;
|
||||
try {
|
||||
const facts = await fetchSearch(endpoint, displayQuery, normalizedLimit, normalizedTimeoutMs, fetchImpl);
|
||||
indexFacts(facts);
|
||||
await writeSearchIndex(searchIndexFile, [...indexedFacts.values()]);
|
||||
recentSearches.set(normalizedQuery, { storedAt: Date.now(), facts });
|
||||
lastRefreshAt = new Date().toISOString();
|
||||
return { facts, complete: true };
|
||||
} catch (error) {
|
||||
searchFailures += 1;
|
||||
upstreamFailures += 1;
|
||||
lastFailure = safeUpstreamFailure(error);
|
||||
lastFailureAt = new Date().toISOString();
|
||||
return { facts: [], complete: false };
|
||||
}
|
||||
});
|
||||
if (!inflight) {
|
||||
inflightSearches.set(normalizedQuery, task);
|
||||
void task.finally(() => inflightSearches.delete(normalizedQuery));
|
||||
}
|
||||
const result = await task;
|
||||
upstreamFacts = result.facts;
|
||||
complete = result.complete;
|
||||
}
|
||||
}
|
||||
const facts = searchIndexedFacts(
|
||||
deduplicateFacts([...localFacts, ...upstreamFacts]),
|
||||
normalizedQuery,
|
||||
normalizedLimit,
|
||||
);
|
||||
const generatedAt = new Date().toISOString();
|
||||
const contentDigest = sha256(JSON.stringify(facts));
|
||||
return {
|
||||
schemaVersion: MAP_REFERENCE_SEARCH_SCHEMA,
|
||||
profileId: TRANSPORT_STATION_PROFILE_ID,
|
||||
sourceRevision: contentDigest,
|
||||
generatedAt,
|
||||
complete,
|
||||
query: displayQuery,
|
||||
contentDigest,
|
||||
facts,
|
||||
metadata: {
|
||||
factCount: facts.length,
|
||||
localMatchCount: localFacts.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function indexFacts(facts) {
|
||||
for (const fact of facts) indexedFacts.set(fact.sourceId, fact);
|
||||
}
|
||||
|
||||
function enqueueFetch(cell) {
|
||||
const inflight = inflightCells.get(cell.key);
|
||||
if (inflight) return inflight;
|
||||
const task = scheduleFetch(async () => {
|
||||
upstreamRequests += 1;
|
||||
try {
|
||||
const document = await fetchCell(endpoint, cell, normalizedTimeoutMs, fetchImpl);
|
||||
await writeCell(cellRoot, cell.key, document);
|
||||
lastRefreshAt = document.generatedAt;
|
||||
return document;
|
||||
} catch (error) {
|
||||
upstreamFailures += 1;
|
||||
lastFailure = safeUpstreamFailure(error);
|
||||
lastFailureAt = new Date().toISOString();
|
||||
return null;
|
||||
}
|
||||
});
|
||||
inflightCells.set(cell.key, task);
|
||||
void task.finally(() => inflightCells.delete(cell.key));
|
||||
return task;
|
||||
}
|
||||
|
||||
function scheduleFetch(run) {
|
||||
return new Promise((resolveTask) => {
|
||||
pendingFetches.push({ run, resolveTask });
|
||||
pumpFetchQueue();
|
||||
});
|
||||
}
|
||||
|
||||
function pumpFetchQueue() {
|
||||
while (activeFetches < MAX_CONCURRENT_UPSTREAM_FETCHES && pendingFetches.length) {
|
||||
const next = pendingFetches.shift();
|
||||
activeFetches += 1;
|
||||
void Promise.resolve()
|
||||
.then(waitForUpstreamStart)
|
||||
.then(next.run)
|
||||
.then(next.resolveTask)
|
||||
.finally(() => {
|
||||
activeFetches -= 1;
|
||||
pumpFetchQueue();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function waitForUpstreamStart() {
|
||||
const turn = upstreamStartQueue.then(async () => {
|
||||
const waitMs = Math.max(0, nextUpstreamStartAt - Date.now());
|
||||
if (waitMs) await new Promise((resolveDelay) => setTimeout(resolveDelay, waitMs));
|
||||
nextUpstreamStartAt = Date.now() + MIN_UPSTREAM_START_INTERVAL_MS;
|
||||
});
|
||||
upstreamStartQueue = turn.catch(() => undefined);
|
||||
return turn;
|
||||
}
|
||||
|
||||
async function status() {
|
||||
let cachedCellCount = 0;
|
||||
try {
|
||||
cachedCellCount = (await readdir(cellRoot)).filter((name) => name.endsWith(".json")).length;
|
||||
} catch {
|
||||
cachedCellCount = 0;
|
||||
}
|
||||
return {
|
||||
profileId: TRANSPORT_STATION_PROFILE_ID,
|
||||
seedRevision: seed.sourceRevision,
|
||||
seedFactCount: seed.facts.length,
|
||||
fetchEnabled,
|
||||
cellDegrees: normalizedCellDegrees,
|
||||
cachedCellCount,
|
||||
upstreamRequests,
|
||||
upstreamFailures,
|
||||
searchRequests,
|
||||
searchFailures,
|
||||
upstreamState: upstreamFailures > 0 && (!lastRefreshAt || Date.parse(lastFailureAt) > Date.parse(lastRefreshAt))
|
||||
? "degraded"
|
||||
: (lastRefreshAt ? "ready" : "idle"),
|
||||
activeFetches,
|
||||
queuedFetches: pendingFetches.length,
|
||||
lastRefreshAt,
|
||||
lastFailure,
|
||||
lastFailureAt,
|
||||
};
|
||||
}
|
||||
|
||||
return Object.freeze({ search, snapshot, status });
|
||||
}
|
||||
|
||||
function validateSeed(value) {
|
||||
if (!isObject(value) || value.schemaVersion !== MAP_REFERENCE_SEED_SCHEMA
|
||||
|| value.profileId !== TRANSPORT_STATION_PROFILE_ID || !Array.isArray(value.facts)
|
||||
|| !value.facts.length || value.facts.length > MAX_FACTS) {
|
||||
throw sourceError("reference_station_seed_invalid");
|
||||
}
|
||||
const facts = value.facts.map(validateFact);
|
||||
if (sha256(JSON.stringify(facts)) !== value.contentDigest) {
|
||||
throw sourceError("reference_station_seed_digest_mismatch");
|
||||
}
|
||||
return Object.freeze({ ...value, facts: Object.freeze(facts) });
|
||||
}
|
||||
|
||||
function validateFact(value) {
|
||||
if (!isObject(value) || !FACT_ID.test(String(value.sourceId || ""))
|
||||
|| !SEMANTIC_TYPES.has(value.semanticType) || !isIso(value.observedAt)
|
||||
|| !isIso(value.receivedAt) || value.presentationStatus !== "active"
|
||||
|| !isObject(value.attributes) || !CATEGORIES.has(value.attributes.category)) {
|
||||
throw sourceError("reference_station_fact_invalid");
|
||||
}
|
||||
const coordinates = value.geometry?.type === "Point" ? value.geometry.coordinates : null;
|
||||
if (!Array.isArray(coordinates) || coordinates.length !== 2
|
||||
|| !Number.isFinite(coordinates[0]) || coordinates[0] < -180 || coordinates[0] > 180
|
||||
|| !Number.isFinite(coordinates[1]) || coordinates[1] < -90 || coordinates[1] > 90) {
|
||||
throw sourceError("reference_station_geometry_invalid");
|
||||
}
|
||||
const allowedAttributes = new Set(["name", "category", "network", "operator", "official_name", "local_name", "uic_ref", "wheelchair"]);
|
||||
if (Object.keys(value.attributes).some((key) => !allowedAttributes.has(key))) {
|
||||
throw sourceError("reference_station_attribute_not_allowed");
|
||||
}
|
||||
if (value.semanticType === "map.terminal" !== (value.attributes.category === "railway_terminal")) {
|
||||
throw sourceError("reference_station_semantic_type_mismatch");
|
||||
}
|
||||
return Object.freeze(structuredClone(value));
|
||||
}
|
||||
|
||||
async function fetchCell(endpoint, cell, timeoutMs, fetchImpl) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const query = `[out:json][timeout:25];(nwr["railway"="station"](${cell.south},${cell.west},${cell.north},${cell.east});nwr["railway"="halt"](${cell.south},${cell.west},${cell.north},${cell.east}););out center tags;`;
|
||||
const response = await fetchImpl(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
||||
"user-agent": "NODE.DC-Map-Gateway/1.0 (+https://node-dc.ru)",
|
||||
},
|
||||
body: new URLSearchParams({ data: query }).toString(),
|
||||
signal: controller.signal,
|
||||
redirect: "error",
|
||||
});
|
||||
if (!response.ok) throw sourceError(`reference_station_upstream_http_${response.status}`);
|
||||
const raw = await response.text();
|
||||
if (Buffer.byteLength(raw) > MAX_CELL_BYTES) throw sourceError("reference_station_upstream_bytes_exceeded");
|
||||
const payload = JSON.parse(raw);
|
||||
if (!Array.isArray(payload?.elements)) throw sourceError("reference_station_upstream_invalid");
|
||||
const observedAt = new Date().toISOString();
|
||||
const facts = payload.elements.flatMap((element) => {
|
||||
try {
|
||||
return [normalizeOverpassElement(element, observedAt)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
return {
|
||||
schemaVersion: "nodedc.map-reference-cell/v1",
|
||||
key: cell.key,
|
||||
generatedAt: observedAt,
|
||||
facts: deduplicateFacts(facts),
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchSearch(endpoint, query, limit, timeoutMs, fetchImpl) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const expression = escapeOverpassString(query);
|
||||
const resultLimit = Math.max(limit, Math.min(96, limit * 3));
|
||||
// Exact OSM name keys are globally indexed by Overpass. Filtering the
|
||||
// whole planet by railway first is several orders of magnitude slower and
|
||||
// routinely times out; station semantics are therefore verified locally
|
||||
// by the same fail-closed normalizer used by viewport cells.
|
||||
const overpassQuery = `[out:json][timeout:25];(nwr["name"="${expression}"];nwr["name:ru"="${expression}"];);out center tags ${resultLimit};`;
|
||||
const response = await fetchImpl(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
|
||||
"user-agent": "NODE.DC-Map-Gateway/1.0 (+https://node-dc.ru)",
|
||||
},
|
||||
body: new URLSearchParams({ data: overpassQuery }).toString(),
|
||||
signal: controller.signal,
|
||||
redirect: "error",
|
||||
});
|
||||
if (!response.ok) throw sourceError(`reference_station_upstream_http_${response.status}`);
|
||||
const raw = await response.text();
|
||||
if (Buffer.byteLength(raw) > MAX_CELL_BYTES) throw sourceError("reference_station_upstream_bytes_exceeded");
|
||||
const payload = JSON.parse(raw);
|
||||
if (!Array.isArray(payload?.elements)) throw sourceError("reference_station_upstream_invalid");
|
||||
const observedAt = new Date().toISOString();
|
||||
return deduplicateFacts(payload.elements.flatMap((element) => {
|
||||
try {
|
||||
return [normalizeOverpassElement(element, observedAt)];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}));
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOverpassElement(value, observedAt) {
|
||||
if (!isObject(value) || !new Set(["node", "way", "relation"]).has(value.type)
|
||||
|| !Number.isSafeInteger(value.id) || value.id <= 0 || !isObject(value.tags)) {
|
||||
throw sourceError("reference_station_upstream_element_invalid");
|
||||
}
|
||||
if (!new Set(["station", "halt"]).has(value.tags.railway)) {
|
||||
throw sourceError("reference_station_upstream_not_station");
|
||||
}
|
||||
const longitude = Number(value.lon ?? value.center?.lon);
|
||||
const latitude = Number(value.lat ?? value.center?.lat);
|
||||
if (!Number.isFinite(longitude) || longitude < -180 || longitude > 180
|
||||
|| !Number.isFinite(latitude) || latitude < -90 || latitude > 90) {
|
||||
throw sourceError("reference_station_upstream_position_invalid");
|
||||
}
|
||||
const name = firstString(value.tags["name:ru"], value.tags.name, value.tags.official_name, value.tags.loc_name);
|
||||
if (!name) throw sourceError("reference_station_upstream_name_required");
|
||||
const category = classifyStation(value.tags, name);
|
||||
return validateFact({
|
||||
sourceId: `osm.${value.type}.${value.id}`,
|
||||
semanticType: category === "railway_terminal" ? "map.terminal" : "map.station",
|
||||
observedAt,
|
||||
receivedAt: observedAt,
|
||||
attributes: compact({
|
||||
name,
|
||||
category,
|
||||
network: optionalString(value.tags.network),
|
||||
operator: optionalString(value.tags.operator),
|
||||
official_name: optionalString(value.tags.official_name),
|
||||
local_name: optionalString(value.tags.loc_name),
|
||||
uic_ref: optionalString(value.tags.uic_ref),
|
||||
wheelchair: optionalString(value.tags.wheelchair),
|
||||
}),
|
||||
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
||||
presentationStatus: "active",
|
||||
});
|
||||
}
|
||||
|
||||
function classifyStation(tags, name) {
|
||||
if (tags.station === "subway" || tags.subway === "yes") return "metro";
|
||||
const terminalText = `${name} ${tags.loc_name || ""} ${tags.official_name || ""}`.toLocaleLowerCase("ru");
|
||||
if (/(?:^|\s)(вокзал|hauptbahnhof|central station|railway station)(?:\s|$)/u.test(terminalText)) {
|
||||
return "railway_terminal";
|
||||
}
|
||||
return "railway_station";
|
||||
}
|
||||
|
||||
function cellsForBbox(bbox, cellDegrees) {
|
||||
const cells = [];
|
||||
const minX = Math.floor(bbox.west / cellDegrees);
|
||||
const maxX = Math.ceil(bbox.east / cellDegrees) - 1;
|
||||
const minY = Math.floor(bbox.south / cellDegrees);
|
||||
const maxY = Math.ceil(bbox.north / cellDegrees) - 1;
|
||||
for (let x = minX; x <= maxX; x += 1) {
|
||||
for (let y = minY; y <= maxY; y += 1) {
|
||||
const west = x * cellDegrees;
|
||||
const south = y * cellDegrees;
|
||||
cells.push({
|
||||
key: `${x}_${y}`,
|
||||
west,
|
||||
south,
|
||||
east: Math.min(180, west + cellDegrees),
|
||||
north: Math.min(90, south + cellDegrees),
|
||||
});
|
||||
}
|
||||
}
|
||||
return cells;
|
||||
}
|
||||
|
||||
function factCoverage(facts, cellDegrees) {
|
||||
const longitudes = facts.map((fact) => fact.geometry.coordinates[0]);
|
||||
const latitudes = facts.map((fact) => fact.geometry.coordinates[1]);
|
||||
return {
|
||||
west: Math.floor(Math.min(...longitudes) / cellDegrees) * cellDegrees,
|
||||
south: Math.floor(Math.min(...latitudes) / cellDegrees) * cellDegrees,
|
||||
east: Math.ceil(Math.max(...longitudes) / cellDegrees) * cellDegrees,
|
||||
north: Math.ceil(Math.max(...latitudes) / cellDegrees) * cellDegrees,
|
||||
};
|
||||
}
|
||||
|
||||
function cellInsideCoverage(cell, coverage) {
|
||||
return cell.west >= coverage.west && cell.east <= coverage.east
|
||||
&& cell.south >= coverage.south && cell.north <= coverage.north;
|
||||
}
|
||||
|
||||
function normalizeBbox(value) {
|
||||
const [west, south, east, north] = Array.isArray(value) ? value.map(Number) : [];
|
||||
if (![west, south, east, north].every(Number.isFinite)
|
||||
|| west < -180 || west >= east || east > 180
|
||||
|| south < -90 || south >= north || north > 90) {
|
||||
throw sourceError("reference_station_bbox_invalid");
|
||||
}
|
||||
return { west, south, east, north };
|
||||
}
|
||||
|
||||
function factInside(fact, bbox) {
|
||||
const [longitude, latitude] = fact.geometry.coordinates;
|
||||
return longitude >= bbox.west && longitude <= bbox.east
|
||||
&& latitude >= bbox.south && latitude <= bbox.north;
|
||||
}
|
||||
|
||||
function deduplicateFacts(values) {
|
||||
return [...new Map(values.map((fact) => [fact.sourceId, validateFact(fact)])).values()]
|
||||
.sort((left, right) => left.sourceId.localeCompare(right.sourceId));
|
||||
}
|
||||
|
||||
function searchIndexedFacts(values, query, limit) {
|
||||
return [...values]
|
||||
.flatMap((fact) => {
|
||||
const rank = bestSearchRank([
|
||||
fact.sourceId,
|
||||
fact.attributes.name,
|
||||
fact.attributes.official_name,
|
||||
fact.attributes.local_name,
|
||||
fact.attributes.uic_ref,
|
||||
], query);
|
||||
return rank === null ? [] : [{ fact, rank }];
|
||||
})
|
||||
.sort((left, right) => (
|
||||
left.rank - right.rank
|
||||
|| left.fact.attributes.name.localeCompare(right.fact.attributes.name, "ru")
|
||||
|| left.fact.sourceId.localeCompare(right.fact.sourceId)
|
||||
))
|
||||
.slice(0, limit)
|
||||
.map(({ fact }) => fact);
|
||||
}
|
||||
|
||||
function bestSearchRank(values, query) {
|
||||
let best = null;
|
||||
for (const value of values) {
|
||||
const normalized = normalizeSearchValue(value);
|
||||
if (!normalized) continue;
|
||||
let rank = null;
|
||||
if (normalized === query) rank = 0;
|
||||
else if (normalized.startsWith(query)) rank = 1;
|
||||
else if (normalized.split(/\s+/u).some((token) => token.startsWith(query))) rank = 2;
|
||||
else if (normalized.includes(query)) rank = 3;
|
||||
if (rank !== null && (best === null || rank < best)) best = rank;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function normalizeSearchQuery(value) {
|
||||
const normalized = normalizeSearchValue(value);
|
||||
if (normalized.length < 2 || normalized.length > 96 || /[\u0000-\u001f\u007f]/u.test(normalized)) {
|
||||
throw sourceError("reference_station_search_query_invalid");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeDisplaySearchQuery(value) {
|
||||
const normalized = String(value ?? "").normalize("NFKC").trim().replace(/\s+/gu, " ");
|
||||
if (normalized.length < 2 || normalized.length > 96 || /[\u0000-\u001f\u007f]/u.test(normalized)) {
|
||||
throw sourceError("reference_station_search_query_invalid");
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function normalizeSearchValue(value) {
|
||||
return String(value ?? "")
|
||||
.normalize("NFKC")
|
||||
.trim()
|
||||
.toLocaleLowerCase("ru")
|
||||
.replace(/\s+/gu, " ");
|
||||
}
|
||||
|
||||
function escapeOverpassString(value) {
|
||||
return String(value).replace(/[\\"]/gu, "\\$&");
|
||||
}
|
||||
|
||||
async function readCell(root, key) {
|
||||
try {
|
||||
const value = JSON.parse(await readFile(withinRoot(root, `${key}.json`), "utf8"));
|
||||
if (value?.schemaVersion !== "nodedc.map-reference-cell/v1" || value.key !== key
|
||||
|| !isIso(value.generatedAt) || !Array.isArray(value.facts) || value.facts.length > MAX_FACTS) return null;
|
||||
return { ...value, facts: value.facts.map(validateFact) };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readSearchIndex(file) {
|
||||
try {
|
||||
const value = JSON.parse(await readFile(file, "utf8"));
|
||||
if (value?.schemaVersion !== "nodedc.map-reference-search-index/v1"
|
||||
|| !Array.isArray(value.facts) || value.facts.length > MAX_FACTS) return [];
|
||||
return value.facts.map(validateFact);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function writeSearchIndex(file, facts) {
|
||||
const normalizedFacts = deduplicateFacts(facts).slice(-MAX_FACTS);
|
||||
const document = {
|
||||
schemaVersion: "nodedc.map-reference-search-index/v1",
|
||||
generatedAt: new Date().toISOString(),
|
||||
facts: normalizedFacts,
|
||||
};
|
||||
const temp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
||||
await writeFile(temp, `${JSON.stringify(document)}\n`, { encoding: "utf8", mode: 0o640 });
|
||||
await rename(temp, file);
|
||||
}
|
||||
|
||||
async function writeCell(root, key, document) {
|
||||
const target = withinRoot(root, `${key}.json`);
|
||||
const temp = withinRoot(root, `${key}.${process.pid}.${Date.now()}.tmp`);
|
||||
await writeFile(temp, `${JSON.stringify(document)}\n`, { encoding: "utf8", mode: 0o640 });
|
||||
await rename(temp, target);
|
||||
}
|
||||
|
||||
function normalizeOverpassEndpoint(value) {
|
||||
let endpoint;
|
||||
try {
|
||||
endpoint = new URL(String(value || ""));
|
||||
} catch {
|
||||
throw sourceError("reference_station_upstream_url_invalid");
|
||||
}
|
||||
if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password
|
||||
|| endpoint.search || endpoint.hash || !endpoint.hostname) {
|
||||
throw sourceError("reference_station_upstream_url_invalid");
|
||||
}
|
||||
return endpoint.toString();
|
||||
}
|
||||
|
||||
function withinRoot(root, child) {
|
||||
const base = resolve(root);
|
||||
const target = resolve(base, child);
|
||||
if (target !== base && !target.startsWith(`${base}${sep}`)) throw sourceError("reference_station_path_escape");
|
||||
return target;
|
||||
}
|
||||
|
||||
function finite(value, min, max, code) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < min || number > max) throw sourceError(code);
|
||||
return number;
|
||||
}
|
||||
|
||||
function firstString(...values) {
|
||||
return values.map(optionalString).find(Boolean);
|
||||
}
|
||||
|
||||
function optionalString(value) {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const normalized = value.trim();
|
||||
return normalized && normalized.length <= 256 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function compact(value) {
|
||||
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined));
|
||||
}
|
||||
|
||||
function isObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isIso(value) {
|
||||
return typeof value === "string" && !Number.isNaN(Date.parse(value));
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function sourceError(code) {
|
||||
const clientError = code === "reference_station_bbox_invalid"
|
||||
|| code === "reference_station_search_query_invalid";
|
||||
return Object.assign(new Error(code), { code, statusCode: clientError ? 400 : 500 });
|
||||
}
|
||||
|
||||
function safeUpstreamFailure(error) {
|
||||
const code = String(error?.message || "");
|
||||
if (/^reference_station_upstream_[a-z0-9_]+$/u.test(code)) return code.slice(0, 96);
|
||||
if (error?.name === "AbortError") return "reference_station_upstream_timeout";
|
||||
return "reference_station_upstream_unavailable";
|
||||
}
|
||||
|
|
@ -7,6 +7,11 @@ import { Readable, Transform } from "node:stream";
|
|||
import { pipeline } from "node:stream/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
MAP_REFERENCE_SEARCH_SCHEMA,
|
||||
TRANSPORT_STATION_PROFILE_ID,
|
||||
createReferenceStationSource,
|
||||
} from "./reference-station-source.mjs";
|
||||
import { loadZoneSourceProfile } from "./zone-source-snapshot.mjs";
|
||||
|
||||
const canonicalCesiumAssetIds = ["1", "2", "96188"];
|
||||
|
|
@ -32,6 +37,14 @@ const zoneSourceRoot = resolve(String(
|
|||
// replaceable: today it is the audited MMap snapshot, later an API adapter can
|
||||
// atomically materialise the same manifest + generation envelope.
|
||||
const zoneSource = await loadZoneSourceProfile(zoneSourceRoot, zoneSourceProfileId);
|
||||
const referenceStationSource = await createReferenceStationSource({
|
||||
seedFile: config.referenceStationSeedFile,
|
||||
cacheDir: config.cacheDir,
|
||||
fetchEnabled: config.referenceStationFetchEnabled,
|
||||
overpassApiBase: config.referenceStationOverpassUrl,
|
||||
cellDegrees: config.referenceStationCellDegrees,
|
||||
timeoutMs: config.upstreamTimeoutMs,
|
||||
});
|
||||
const liveCache = createCacheStore("live", config.cacheDir, true);
|
||||
const offlineSnapshot = config.offlineSnapshotDir ? createCacheStore("offline-snapshot", config.offlineSnapshotDir, false) : null;
|
||||
const inflightWrites = new Map();
|
||||
|
|
@ -70,6 +83,15 @@ const server = createServer(async (request, response) => {
|
|||
}
|
||||
if (!isAllowedRequest(request)) return writeJson(response, 401, { ok: false, error: "map_gateway_auth_required" });
|
||||
|
||||
if (requestUrl.pathname === `/api/map/reference-sources/v1/profiles/${TRANSPORT_STATION_PROFILE_ID}/current`
|
||||
&& ["GET", "HEAD"].includes(request.method || "")) {
|
||||
return await serveReferenceStationSnapshot(request, response, requestUrl);
|
||||
}
|
||||
if (requestUrl.pathname === `/api/map/reference-sources/v1/profiles/${TRANSPORT_STATION_PROFILE_ID}/search`
|
||||
&& ["GET", "HEAD"].includes(request.method || "")) {
|
||||
return await serveReferenceStationSearch(request, response, requestUrl);
|
||||
}
|
||||
|
||||
if (requestUrl.pathname === "/healthz" && request.method === "GET") {
|
||||
return writeJson(response, 200, {
|
||||
ok: true,
|
||||
|
|
@ -89,6 +111,9 @@ const server = createServer(async (request, response) => {
|
|||
sourceZoneCount: zoneSource.generation.metadata.sourceZoneCount,
|
||||
publishedZoneCount: zoneSource.generation.metadata.publishedZoneCount,
|
||||
},
|
||||
referenceSources: {
|
||||
transportStations: await referenceStationSource.status(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -126,6 +151,7 @@ server.listen(config.port, "0.0.0.0", () => {
|
|||
console.log(`Live map cache: ${config.cacheDir} (${config.mode}, max ${Math.round(config.maxCacheBytes / 1024 / 1024)} MB)`);
|
||||
if (offlineSnapshot) console.log(`Offline map snapshot: ${config.offlineSnapshotDir} (read-only)`);
|
||||
console.log(`Zone source: ${zoneSourceProfileId}@${zoneSource.generation.sourceRevision} (${zoneSource.generation.metadata.publishedZoneCount} zones)`);
|
||||
console.log(`Map reference source: ${TRANSPORT_STATION_PROFILE_ID}`);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => server.close());
|
||||
|
|
@ -146,6 +172,50 @@ function serveZoneSourceGeneration(request, response) {
|
|||
return response.end(request.method === "HEAD" ? undefined : zoneSource.body);
|
||||
}
|
||||
|
||||
async function serveReferenceStationSnapshot(request, response, requestUrl) {
|
||||
const rawBbox = requestUrl.searchParams.get("bbox");
|
||||
const bbox = rawBbox ? rawBbox.split(",").map(Number) : undefined;
|
||||
const snapshot = await referenceStationSource.snapshot({ bbox });
|
||||
const etag = `"sha256:${snapshot.contentDigest}"`;
|
||||
if (request.headers["if-none-match"] === etag) {
|
||||
response.writeHead(304, { ETag: etag, "Cache-Control": "private, max-age=300" });
|
||||
return response.end();
|
||||
}
|
||||
const body = Buffer.from(`${JSON.stringify(snapshot)}\n`);
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Content-Length": body.byteLength,
|
||||
"Cache-Control": "private, max-age=300",
|
||||
ETag: etag,
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
});
|
||||
return response.end(request.method === "HEAD" ? undefined : body);
|
||||
}
|
||||
|
||||
async function serveReferenceStationSearch(request, response, requestUrl) {
|
||||
const search = await referenceStationSource.search({
|
||||
query: requestUrl.searchParams.get("q"),
|
||||
limit: requestUrl.searchParams.get("limit"),
|
||||
});
|
||||
if (search.schemaVersion !== MAP_REFERENCE_SEARCH_SCHEMA) {
|
||||
throw new Error("reference_station_search_contract_invalid");
|
||||
}
|
||||
const etag = `"sha256:${search.contentDigest}"`;
|
||||
if (request.headers["if-none-match"] === etag) {
|
||||
response.writeHead(304, { ETag: etag, "Cache-Control": "private, max-age=300" });
|
||||
return response.end();
|
||||
}
|
||||
const body = Buffer.from(`${JSON.stringify(search)}\n`);
|
||||
response.writeHead(200, {
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"Content-Length": body.byteLength,
|
||||
"Cache-Control": "private, max-age=300",
|
||||
ETag: etag,
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
});
|
||||
return response.end(request.method === "HEAD" ? undefined : body);
|
||||
}
|
||||
|
||||
async function readConfig() {
|
||||
const mode = String(process.env.MAP_CACHE_MODE || "readwrite").trim().toLowerCase();
|
||||
if (!["readwrite", "readonly", "offline"].includes(mode)) throw new Error("invalid_map_cache_mode");
|
||||
|
|
@ -187,6 +257,13 @@ async function readConfig() {
|
|||
upstreamAllowlist: new Set(allowlist),
|
||||
legacyCacheHosts: new Set(parseList(process.env.MAP_GATEWAY_LEGACY_CACHE_HOSTS || "")),
|
||||
offlineProviderAllowlist: new Set(parseList(process.env.MAP_GATEWAY_OFFLINE_PROVIDER_ALLOWLIST || "")),
|
||||
referenceStationSeedFile: String(
|
||||
process.env.MAP_REFERENCE_STATION_SEED_FILE
|
||||
|| resolve(dirname(fileURLToPath(import.meta.url)), "../reference-sources/transport-stations/moscow-v1.json"),
|
||||
).trim(),
|
||||
referenceStationFetchEnabled: parseBoolean(process.env.MAP_REFERENCE_STATION_FETCH_ENABLED, true),
|
||||
referenceStationCellDegrees: Number(process.env.MAP_REFERENCE_STATION_CELL_DEGREES || 0.5),
|
||||
referenceStationOverpassUrl: String(process.env.MAP_REFERENCE_STATION_OVERPASS_URL || "https://overpass-api.de/api/interpreter").trim(),
|
||||
corsOrigins: new Set(parseList(process.env.MAP_GATEWAY_CORS_ORIGIN || "http://127.0.0.1:3333,http://localhost:3333")),
|
||||
allowAnonymous: parseBoolean(process.env.MAP_GATEWAY_ALLOW_ANONYMOUS, process.env.NODE_ENV !== "production"),
|
||||
trustedSubjectHeader: String(process.env.MAP_GATEWAY_TRUSTED_SUBJECT_HEADER || "x-nodedc-user-id").toLowerCase(),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,204 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
MAP_REFERENCE_SEARCH_SCHEMA,
|
||||
MAP_REFERENCE_SNAPSHOT_SCHEMA,
|
||||
TRANSPORT_STATION_PROFILE_ID,
|
||||
createReferenceStationSource,
|
||||
} from "../src/reference-station-source.mjs";
|
||||
|
||||
const root = resolve(fileURLToPath(new URL("..", import.meta.url)));
|
||||
const seedFile = join(root, "reference-sources", "transport-stations", "moscow-v1.json");
|
||||
|
||||
test("packaged transport station profile is normalized and provider-neutral", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
const source = await createReferenceStationSource({ seedFile, cacheDir, fetchEnabled: false });
|
||||
const snapshot = await source.snapshot();
|
||||
assert.equal(snapshot.schemaVersion, MAP_REFERENCE_SNAPSHOT_SCHEMA);
|
||||
assert.equal(snapshot.profileId, TRANSPORT_STATION_PROFILE_ID);
|
||||
assert.equal(snapshot.complete, true);
|
||||
assert.equal(snapshot.metadata.factCount, 638);
|
||||
assert.deepEqual(snapshot.metadata.categoryCounts, {
|
||||
metro: 273,
|
||||
railway_station: 357,
|
||||
railway_terminal: 8,
|
||||
});
|
||||
assert.ok(snapshot.facts.every((fact) => !JSON.stringify(fact).match(/https?:|token|credential|endpoint/i)));
|
||||
assert.ok(snapshot.facts.every((fact) => ["map.station", "map.terminal"].includes(fact.semanticType)));
|
||||
});
|
||||
|
||||
test("bbox inside the seed spatial-cell coverage never waits for upstream", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let calls = 0;
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
cellDegrees: 0.5,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
throw new Error("seed coverage must not call upstream");
|
||||
},
|
||||
});
|
||||
const snapshot = await source.snapshot({ bbox: [36.9, 55.4, 38.2, 56.1] });
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(snapshot.complete, true);
|
||||
assert.equal(snapshot.metadata.requestedCellCount, 12);
|
||||
assert.ok(snapshot.facts.length > 0);
|
||||
});
|
||||
|
||||
test("bbox fetch uses one cached spatial cell and never exposes upstream payload", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let calls = 0;
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
cellDegrees: 1,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
return new Response(JSON.stringify({
|
||||
elements: [{
|
||||
type: "node",
|
||||
id: 987654321,
|
||||
lat: 59.93,
|
||||
lon: 30.31,
|
||||
tags: { railway: "station", station: "subway", name: "Тестовая", website: "https://not-projected.invalid" },
|
||||
}],
|
||||
}), { status: 200, headers: { "content-type": "application/json" } });
|
||||
},
|
||||
});
|
||||
const first = await source.snapshot({ bbox: [30, 59, 31, 60] });
|
||||
const second = await source.snapshot({ bbox: [30, 59, 31, 60] });
|
||||
assert.equal(calls, 1);
|
||||
assert.ok(first.facts.some((fact) => fact.sourceId === "osm.node.987654321"));
|
||||
assert.deepEqual(first.facts, second.facts);
|
||||
assert.ok(!JSON.stringify(first).includes("not-projected"));
|
||||
const cached = JSON.parse(await readFile(join(cacheDir, "reference-features", TRANSPORT_STATION_PROFILE_ID, "30_59.json"), "utf8"));
|
||||
assert.equal(cached.schemaVersion, "nodedc.map-reference-cell/v1");
|
||||
});
|
||||
|
||||
test("viewport cells use bounded concurrency and deduplicate inflight fetches", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let active = 0;
|
||||
let maximumActive = 0;
|
||||
let calls = 0;
|
||||
const starts = [];
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
cellDegrees: 0.5,
|
||||
fetchImpl: async () => {
|
||||
calls += 1;
|
||||
starts.push(Date.now());
|
||||
active += 1;
|
||||
maximumActive = Math.max(maximumActive, active);
|
||||
await new Promise((resolveDelay) => setTimeout(resolveDelay, 15));
|
||||
active -= 1;
|
||||
return new Response(JSON.stringify({ elements: [] }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
},
|
||||
});
|
||||
const bbox = [29.9, 59.7, 30.9, 60.2];
|
||||
await Promise.all([source.snapshot({ bbox }), source.snapshot({ bbox })]);
|
||||
assert.equal(calls, 6);
|
||||
assert.ok(maximumActive <= 2);
|
||||
assert.ok(starts.slice(1).every((startedAt, index) => startedAt - starts[index] >= 650));
|
||||
const status = await source.status();
|
||||
assert.equal(status.activeFetches, 0);
|
||||
assert.equal(status.queuedFetches, 0);
|
||||
assert.equal(status.upstreamState, "ready");
|
||||
});
|
||||
|
||||
test("Russian name wins and upstream failures are safe diagnostics", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let fail = false;
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
cellDegrees: 1,
|
||||
fetchImpl: async () => {
|
||||
if (fail) throw new TypeError("getaddrinfo EAI_AGAIN overpass.internal.example");
|
||||
return new Response(JSON.stringify({
|
||||
elements: [{
|
||||
type: "node",
|
||||
id: 987654322,
|
||||
lat: 59.93,
|
||||
lon: 30.31,
|
||||
tags: {
|
||||
railway: "station",
|
||||
station: "subway",
|
||||
name: "Petrogradskaya",
|
||||
"name:ru": "Петроградская",
|
||||
},
|
||||
}],
|
||||
}), { status: 200, headers: { "content-type": "application/json" } });
|
||||
},
|
||||
});
|
||||
const snapshot = await source.snapshot({ bbox: [30, 59, 31, 60] });
|
||||
assert.equal(snapshot.facts[0].attributes.name, "Петроградская");
|
||||
|
||||
fail = true;
|
||||
await source.snapshot({ bbox: [31, 59, 32, 60] });
|
||||
const status = await source.status();
|
||||
assert.equal(status.upstreamState, "degraded");
|
||||
assert.equal(status.lastFailure, "reference_station_upstream_unavailable");
|
||||
assert.ok(status.lastFailureAt);
|
||||
assert.ok(!JSON.stringify(status).includes("overpass.internal.example"));
|
||||
});
|
||||
|
||||
test("global station search is normalized, deduplicated and persisted", async () => {
|
||||
const cacheDir = await mkdtemp(join(tmpdir(), "nodedc-reference-stations-"));
|
||||
let calls = 0;
|
||||
const source = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
fetchImpl: async (_url, options) => {
|
||||
calls += 1;
|
||||
const body = new URLSearchParams(options.body);
|
||||
assert.match(body.get("data"), /Петроградская/u);
|
||||
return new Response(JSON.stringify({
|
||||
elements: [{
|
||||
type: "node",
|
||||
id: 987654323,
|
||||
lat: 59.96639,
|
||||
lon: 30.3113,
|
||||
tags: {
|
||||
railway: "station",
|
||||
station: "subway",
|
||||
name: "Petrogradskaya",
|
||||
"name:ru": "Петроградская",
|
||||
website: "https://not-projected.invalid",
|
||||
},
|
||||
}],
|
||||
}), { status: 200, headers: { "content-type": "application/json" } });
|
||||
},
|
||||
});
|
||||
const first = await source.search({ query: "Петроградская", limit: 8 });
|
||||
const second = await source.search({ query: "Петроградская", limit: 8 });
|
||||
assert.equal(first.schemaVersion, MAP_REFERENCE_SEARCH_SCHEMA);
|
||||
assert.equal(first.complete, true);
|
||||
assert.equal(calls, 1);
|
||||
assert.deepEqual(first.facts, second.facts);
|
||||
assert.equal(first.facts[0].attributes.name, "Петроградская");
|
||||
assert.ok(!JSON.stringify(first).includes("not-projected"));
|
||||
const persisted = JSON.parse(await readFile(
|
||||
join(cacheDir, "reference-features", `${TRANSPORT_STATION_PROFILE_ID}.search.json`),
|
||||
"utf8",
|
||||
));
|
||||
assert.equal(persisted.schemaVersion, "nodedc.map-reference-search-index/v1");
|
||||
|
||||
const restarted = await createReferenceStationSource({
|
||||
seedFile,
|
||||
cacheDir,
|
||||
fetchEnabled: false,
|
||||
});
|
||||
const cached = await restarted.search({ query: "Петроградская", limit: 8 });
|
||||
assert.equal(cached.complete, true);
|
||||
assert.equal(cached.facts[0].sourceId, "osm.node.987654323");
|
||||
});
|
||||
Loading…
Reference in New Issue