Add SEO mode project workflow
This commit is contained in:
parent
20cf1a40f0
commit
d27e6ee43d
|
|
@ -0,0 +1,36 @@
|
|||
# Dependencies
|
||||
node_modules/
|
||||
|
||||
# Builds
|
||||
dist/
|
||||
build/
|
||||
.vite/
|
||||
|
||||
# Environment and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Local runtime data
|
||||
data/
|
||||
tmp/
|
||||
temp/
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# Local infrastructure volumes
|
||||
.postgres/
|
||||
.minio/
|
||||
|
||||
# Unpacked archive references kept local; source archives stay tracked.
|
||||
/SEO-farm/
|
||||
/seo_mode_test_site/
|
||||
|
||||
# Nested VCS metadata from unpacked archives
|
||||
.git/
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
SERVER_PORT=4100
|
||||
DATABASE_URL=postgres://seo_farm:seo_farm@localhost:5432/seo_farm
|
||||
|
||||
S3_ENDPOINT=http://localhost:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_ACCESS_KEY_ID=seo_farm
|
||||
S3_SECRET_ACCESS_KEY=seo_farm_secret
|
||||
S3_BUCKET=seo-farm-artifacts
|
||||
S3_FORCE_PATH_STYLE=true
|
||||
|
||||
WORDSTAT_PROVIDER=disabled
|
||||
# WORDSTAT_PROVIDER=mcp_kv
|
||||
# WORDSTAT_MCP_ENDPOINT=http://localhost:4120/wordstat/collect
|
||||
# WORDSTAT_PROVIDER=yandex_api
|
||||
# WORDSTAT_API_BASE_URL=https://searchapi.api.cloud.yandex.net/v2/wordstat
|
||||
# WORDSTAT_API_TOKEN=change-me
|
||||
# WORDSTAT_AUTH_TYPE=api_key
|
||||
# WORDSTAT_FOLDER_ID=b1g...
|
||||
# WORDSTAT_REGION_IDS=225
|
||||
# WORDSTAT_DEVICES=DEVICE_ALL
|
||||
# WORDSTAT_NUM_PHRASES=50
|
||||
|
||||
SERP_PROVIDER=disabled
|
||||
# SERP_API_BASE_URL=https://serp-provider.internal
|
||||
# SERP_API_TOKEN=change-me
|
||||
|
||||
VITE_API_BASE_URL=http://localhost:4100
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
# Dependencies
|
||||
node_modules/
|
||||
|
||||
# Builds
|
||||
dist/
|
||||
build/
|
||||
.vite/
|
||||
|
||||
# Environment and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Local runtime data
|
||||
data/
|
||||
tmp/
|
||||
temp/
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# Local infrastructure volumes
|
||||
.postgres/
|
||||
.minio/
|
||||
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# seo_mode Workspace
|
||||
|
||||
Рабочая папка для разработки seo_mode.
|
||||
|
||||
Архитектурная документация лежит в `SEO-farm/`.
|
||||
|
||||
## Текущий статус
|
||||
|
||||
Код приложения уже инициализирован.
|
||||
|
||||
Готов текущий MVP-срез:
|
||||
|
||||
- React/Vite UI и Node.js backend.
|
||||
- Postgres + MinIO через `infra/docker-compose.yml`.
|
||||
- Миграции базовой проектной модели.
|
||||
- Создание, переименование, копирование и удаление проектов.
|
||||
- Импорт папки через browser folder picker / drag-and-drop в MinIO.
|
||||
- Скан проекта, `scan_version`, stable `pages/media`, повторный scan.
|
||||
- `project-index.json` как JSON-снимок скана в MinIO.
|
||||
- Baseline audit по выбранным страницам: title, description, H1/headings, видимый текст, media/ALT/filename.
|
||||
- UI выбора страниц деревом по путям, severity-фильтры, media issue list и просмотр ошибок только по выбранным страницам.
|
||||
- Page Workspace MVP: preview страницы, ошибки выбранной страницы, логические секции, section editor, save/reset без изменения исходного проекта.
|
||||
|
||||
Текущий этап: довести `Page Workspace / Draft Model` до кликабельного редакторского сценария и подготовить следующий шаг `Model Provider / Codex Exec`.
|
||||
|
||||
## Базовые решения
|
||||
|
||||
- `Postgres` — основной источник истины.
|
||||
- `MinIO/S3` — хранилище raw artifacts, snapshots, reports, backups и exports.
|
||||
- `browser_folder` — текущий рабочий source adapter для импорта папки в MinIO и будущей hosted-версии.
|
||||
- `local_folder` — dev/local adapter для ручного абсолютного пути.
|
||||
- `scan_version + reconcile` — модель повторного скана без потери утверждённых решений.
|
||||
- `Apply` проходит через changeset, dry-run, backup и rollback.
|
||||
|
||||
## Структура
|
||||
|
||||
```text
|
||||
app/ React/Vite UI
|
||||
server/ Node.js backend
|
||||
infra/ Postgres + MinIO local infrastructure
|
||||
SEO-farm/ архитектурная документация
|
||||
```
|
||||
|
||||
## Запуск
|
||||
|
||||
```powershell
|
||||
npm install
|
||||
Copy-Item .env.example .env
|
||||
docker compose -f infra/docker-compose.yml up -d
|
||||
npm run db:migrate
|
||||
npm run dev
|
||||
```
|
||||
|
||||
Если Docker Desktop не установлен, backend можно запустить без deep health checks:
|
||||
|
||||
```powershell
|
||||
npm run dev:server
|
||||
```
|
||||
|
|
@ -0,0 +1,636 @@
|
|||
# checkOverseo Specification
|
||||
|
||||
## 1. Цель
|
||||
|
||||
`checkOverseo` — локальный инструмент проверки перетошноты, заспамленности, водности и ключевых повторов в SEO-правках.
|
||||
|
||||
Это не платная интеграция и не внешний API. Мы используем Text.ru только как референс по смыслу показателей и устройству SEO-анализа текста.
|
||||
|
||||
Главный вопрос инструмента:
|
||||
|
||||
```text
|
||||
Текст нормально оптимизирован или уже переблеван ключами?
|
||||
```
|
||||
|
||||
## 2. Референс: что берём из Text.ru как идею
|
||||
|
||||
Из Text.ru SEO-анализатора берём не API, а логику показателей:
|
||||
|
||||
- количество символов;
|
||||
- количество слов;
|
||||
- процент водности;
|
||||
- процент заспамленности;
|
||||
- список ключей;
|
||||
- группы ключей;
|
||||
- смешанные слова из разных алфавитов;
|
||||
- подсветка проблемных повторов, если получится реализовать.
|
||||
|
||||
Источники для ориентира:
|
||||
|
||||
- Text.ru API/check description: https://text.ru/api-check
|
||||
- Text.ru SEO-анализ: https://text.ru/seo
|
||||
- Text.ru FAQ SEO-анализ: https://text.ru/faq/instrukcii/seo-analiz-teksta
|
||||
|
||||
Важно: Text.ru — только ориентир. Мы не подключаем их API и не платим за него.
|
||||
|
||||
## 2.1. Разбор публичной модели Text.ru
|
||||
|
||||
По публичному описанию Text.ru SEO-анализатор устроен как набор параллельных проверок текста:
|
||||
|
||||
```text
|
||||
raw text
|
||||
|
|
||||
+-- character/word counter
|
||||
+-- keyword extractor
|
||||
+-- keyword group builder
|
||||
+-- water detector
|
||||
+-- spam detector
|
||||
+-- mixed alphabet detector
|
||||
+-- highlighter / word position mapper
|
||||
```
|
||||
|
||||
### 2.1.1. Базовые счётчики
|
||||
|
||||
Text.ru считает:
|
||||
|
||||
```text
|
||||
count_chars_with_space
|
||||
count_chars_without_space
|
||||
count_words
|
||||
```
|
||||
|
||||
Нам нужно повторить это локально на каждом scope:
|
||||
|
||||
```text
|
||||
page
|
||||
section
|
||||
title
|
||||
description
|
||||
heading
|
||||
alt
|
||||
filename
|
||||
```
|
||||
|
||||
### 2.1.2. Ключи
|
||||
|
||||
Text.ru описывает поиск поисковых ключей в тексте, количество вхождений и морфологические варианты. В API manual это выражено как:
|
||||
|
||||
```text
|
||||
list_keys:
|
||||
key_title
|
||||
count
|
||||
```
|
||||
|
||||
Где `count` — количество вхождений ключа во всех формах.
|
||||
|
||||
Наша локальная версия:
|
||||
|
||||
```text
|
||||
known_keys:
|
||||
- берём из keywords.yml
|
||||
- exact phrase
|
||||
- soft_forms
|
||||
|
||||
detected_keys:
|
||||
- считаем частотные unigram/bigram/trigram
|
||||
- отбрасываем служебный мусор правилами
|
||||
- показываем как "detected", не как утверждённый SEO-ключ
|
||||
```
|
||||
|
||||
Важно: мы не обязаны идеально повторять морфологию Text.ru. Для MVP достаточно:
|
||||
|
||||
```text
|
||||
exact phrase
|
||||
+ manually/model-defined soft_forms
|
||||
+ detected repeated phrases
|
||||
```
|
||||
|
||||
### 2.1.3. Группы ключей
|
||||
|
||||
Text.ru группирует ключи по составу слов и сортирует по числу значимых слов. В API manual это:
|
||||
|
||||
```text
|
||||
list_keys_group:
|
||||
key_title
|
||||
count
|
||||
sub_keys:
|
||||
key_title
|
||||
count
|
||||
```
|
||||
|
||||
Пример идеи:
|
||||
|
||||
```text
|
||||
"check text" -> group
|
||||
"check"
|
||||
"text"
|
||||
```
|
||||
|
||||
Наша локальная версия:
|
||||
|
||||
```text
|
||||
keyword group:
|
||||
parent phrase:
|
||||
- full phrase
|
||||
- single-word subkeys
|
||||
- soft_forms
|
||||
- repeated close phrases
|
||||
```
|
||||
|
||||
Зачем это нужно:
|
||||
|
||||
```text
|
||||
- видеть, что текст не только повторяет один точный ключ;
|
||||
- видеть, что одна группа терминов забила секцию;
|
||||
- ловить ситуацию, когда разные формы одного смысла всё равно создают перетошноту.
|
||||
```
|
||||
|
||||
### 2.1.4. Водность
|
||||
|
||||
Text.ru определяет воду как процент стоп-слов, фразеологизмов, оборотов, соединительных слов и фраз, которые не несут смысловой нагрузки.
|
||||
|
||||
Публичные пороги:
|
||||
|
||||
```text
|
||||
до 15% естественное содержание воды
|
||||
15-30% превышенное содержание воды
|
||||
от 30% высокое содержание воды
|
||||
```
|
||||
|
||||
Наша локальная версия не должна быть обычным stopword-remover. Нам нужен редакторский сигнал:
|
||||
|
||||
```text
|
||||
water_score =
|
||||
служебные слова
|
||||
+ слабые бизнес-фразы
|
||||
+ общие SEO-обороты
|
||||
+ абстрактные фразы без объекта
|
||||
+ предложения без конкретного действия/сущности
|
||||
```
|
||||
|
||||
То есть мы считаем приблизительный процент, но в отчёте показываем не только число, а конкретные причины.
|
||||
|
||||
### 2.1.5. Заспамленность
|
||||
|
||||
Text.ru описывает заспамленность как количество поисковых ключевых слов в тексте.
|
||||
|
||||
Публичные пороги:
|
||||
|
||||
```text
|
||||
до 30% естественное содержание ключевых слов
|
||||
30-60% оптимизированный текст
|
||||
от 60% сильно оптимизированный / заспамленный текст
|
||||
```
|
||||
|
||||
Наша локальная версия считает это по двум слоям:
|
||||
|
||||
```text
|
||||
1. known-key spam:
|
||||
- primary keywords
|
||||
- secondary keywords
|
||||
- soft_forms
|
||||
|
||||
2. detected-key spam:
|
||||
- частотные слова
|
||||
- частотные биграммы
|
||||
- частотные триграммы
|
||||
- группы похожих ключей
|
||||
```
|
||||
|
||||
Для нас важнее не общий процент по странице, а риск по секции:
|
||||
|
||||
```text
|
||||
section spam > page spam
|
||||
```
|
||||
|
||||
Потому что лендинг может быть нормальным целиком, но отдельная карточка будет переблевана ключом.
|
||||
|
||||
### 2.1.6. Mixed words
|
||||
|
||||
Text.ru ищет слова, где смешаны символы разных алфавитов со схожим написанием.
|
||||
|
||||
Наша локальная версия:
|
||||
|
||||
```text
|
||||
mixedWordsDetector:
|
||||
- кириллица + латиница внутри одного слова
|
||||
- похожие символы: а/a, о/o, е/e, р/p, с/c, х/x
|
||||
- выдавать позиции и snippets
|
||||
```
|
||||
|
||||
Это не SEO-ключи, но полезная техническая проверка качества текста.
|
||||
|
||||
### 2.1.7. Подсветка и позиции
|
||||
|
||||
В Text.ru API есть режимы, где возвращаются:
|
||||
|
||||
```text
|
||||
words
|
||||
text_view
|
||||
words_pos
|
||||
```
|
||||
|
||||
Это значит, что их интерфейс может подсвечивать проблемные слова/ключи по позициям.
|
||||
|
||||
Наша локальная версия должна с самого начала хранить offsets:
|
||||
|
||||
```ts
|
||||
type Token = {
|
||||
text: string;
|
||||
normalized: string;
|
||||
start: number;
|
||||
end: number;
|
||||
sentenceIndex: number;
|
||||
paragraphIndex: number;
|
||||
};
|
||||
```
|
||||
|
||||
Зачем:
|
||||
|
||||
```text
|
||||
- подсветить ключ в UI;
|
||||
- показать proximity;
|
||||
- показать повтор рядом;
|
||||
- показать water phrase;
|
||||
- показать mixed word;
|
||||
- дать Codex точный snippet для правки.
|
||||
```
|
||||
|
||||
## 3. Архитектура
|
||||
|
||||
```text
|
||||
checkOverseo
|
||||
|
|
||||
+-- TextNormalizer
|
||||
| +-- normalize spaces
|
||||
| +-- normalize quotes/dashes where needed
|
||||
| +-- lowercase for counting
|
||||
| +-- keep original text for report snippets
|
||||
|
|
||||
+-- Tokenizer
|
||||
| +-- words
|
||||
| +-- sentences
|
||||
| +-- bigrams
|
||||
| +-- trigrams
|
||||
|
|
||||
+-- KeywordCounter
|
||||
| +-- exact keywords
|
||||
| +-- soft_forms from keywords.yml
|
||||
| +-- key groups
|
||||
|
|
||||
+-- NauseaAnalyzer
|
||||
| +-- classic nausea
|
||||
| +-- academic nausea
|
||||
| +-- phrase repetition
|
||||
|
|
||||
+-- WaterAnalyzer
|
||||
| +-- local water-word dictionary
|
||||
| +-- weak phrase markers
|
||||
| +-- business-cliche markers
|
||||
|
|
||||
+-- ProximityAnalyzer
|
||||
| +-- same keyword too close
|
||||
| +-- repeated key in neighboring sentences
|
||||
|
|
||||
+-- ScopeAnalyzer
|
||||
| +-- page
|
||||
| +-- section
|
||||
| +-- title
|
||||
| +-- description
|
||||
| +-- h1/h2
|
||||
| +-- alt
|
||||
| +-- filename
|
||||
|
|
||||
+-- ReportBuilder
|
||||
+-- warnings
|
||||
+-- scores
|
||||
+-- snippets
|
||||
+-- verdict
|
||||
```
|
||||
|
||||
## 4. Вход
|
||||
|
||||
```ts
|
||||
type CheckOverseoInput = {
|
||||
text: string;
|
||||
originalText?: string;
|
||||
scope: "page" | "section" | "title" | "description" | "heading" | "alt" | "filename";
|
||||
sourceId: string;
|
||||
sectionId?: string;
|
||||
keywords: KeywordEntry[];
|
||||
pageMap: PageMap;
|
||||
options?: CheckOverseoOptions;
|
||||
};
|
||||
```
|
||||
|
||||
## 5. Источник правил
|
||||
|
||||
### 5.1. `keywords.yml`
|
||||
|
||||
```yaml
|
||||
keywords:
|
||||
- phrase: "операционная модель бизнеса"
|
||||
type: "primary"
|
||||
section: "hero"
|
||||
exact_limit: 1
|
||||
max_section_count: 2
|
||||
soft_forms:
|
||||
- "операционная модель"
|
||||
- "модель бизнеса"
|
||||
status: "use"
|
||||
```
|
||||
|
||||
### 5.2. `page-map.yml`
|
||||
|
||||
```yaml
|
||||
page:
|
||||
primary_intent: "главный смысл страницы"
|
||||
forbidden_intents: []
|
||||
|
||||
sections:
|
||||
hero:
|
||||
meaning: "смысл секции"
|
||||
primary_keywords: []
|
||||
secondary_keywords: []
|
||||
max_lines_hint: 6
|
||||
```
|
||||
|
||||
### 5.3. Local dictionaries
|
||||
|
||||
```text
|
||||
data/rules/
|
||||
water-words-ru.yml
|
||||
weak-business-phrases.yml
|
||||
negative-keyword-intents.yml
|
||||
safe-technical-terms.yml
|
||||
```
|
||||
|
||||
Словари нужны не для автоматического переписывания, а для предупреждений.
|
||||
|
||||
## 6. Проверки
|
||||
|
||||
### 6.1. Counts
|
||||
|
||||
```text
|
||||
- chars_with_space
|
||||
- chars_without_space
|
||||
- words_count
|
||||
- sentences_count
|
||||
- average_sentence_length
|
||||
```
|
||||
|
||||
### 6.2. Keyword counts
|
||||
|
||||
```text
|
||||
- exact keyword count
|
||||
- soft_forms count
|
||||
- primary keyword count
|
||||
- secondary keyword count
|
||||
- key group count
|
||||
- keyword density by page
|
||||
- keyword density by section
|
||||
```
|
||||
|
||||
### 6.3. Classic nausea
|
||||
|
||||
Классическая тошнота считается как корень из максимальной частоты самого повторяемого слова или ключевой фразы.
|
||||
|
||||
```text
|
||||
classic_nausea = sqrt(max_term_frequency)
|
||||
```
|
||||
|
||||
Используется как локальный индикатор частого повторения.
|
||||
|
||||
### 6.4. Academic nausea
|
||||
|
||||
Академическая тошнота считается как доля самого частого слова/термина в общем количестве слов.
|
||||
|
||||
```text
|
||||
academic_nausea_percent = max_term_frequency / words_count * 100
|
||||
```
|
||||
|
||||
Также считаем отдельно:
|
||||
|
||||
```text
|
||||
keyword_academic_nausea_percent
|
||||
```
|
||||
|
||||
Чтобы видеть не просто частое служебное слово, а именно перебор нужного нам ключа.
|
||||
|
||||
### 6.5. Phrase repetition
|
||||
|
||||
```text
|
||||
- repeated exact phrase
|
||||
- repeated bigram
|
||||
- repeated trigram
|
||||
- same sentence starts
|
||||
- repeated sentence skeleton
|
||||
```
|
||||
|
||||
### 6.6. Proximity
|
||||
|
||||
```text
|
||||
- один и тот же ключ повторяется в соседних предложениях;
|
||||
- один и тот же ключ повторяется в соседних абзацах;
|
||||
- primary keyword стоит слишком близко к прошлому вхождению;
|
||||
- alt повторяет heading без причины;
|
||||
- filename повторяет alt и ключ без смысла.
|
||||
```
|
||||
|
||||
### 6.7. Water score
|
||||
|
||||
Локальная водность — не попытка идеально повторить Text.ru. Это наш сигнал:
|
||||
|
||||
```text
|
||||
water_score = water_words_count / words_count * 100
|
||||
```
|
||||
|
||||
Плюс отдельные предупреждения:
|
||||
|
||||
```text
|
||||
- слабые бизнес-фразы;
|
||||
- общие SEO-фразы;
|
||||
- много абстрактных существительных;
|
||||
- длинные предложения без конкретного объекта;
|
||||
- фразы, которые не добавляют смысла.
|
||||
```
|
||||
|
||||
### 6.8. Scope rules
|
||||
|
||||
Разные зоны проверяются по-разному:
|
||||
|
||||
```text
|
||||
page:
|
||||
- общий риск заспамленности
|
||||
- общий список ключей
|
||||
|
||||
section:
|
||||
- соответствие ключей секции
|
||||
- локальный перебор
|
||||
- конфликт с meaning секции
|
||||
|
||||
title/description:
|
||||
- нет ли повторения одного ключа дважды
|
||||
- нет ли набора ключей вместо нормального сниппета
|
||||
|
||||
heading:
|
||||
- нет ли keyword stuffing
|
||||
- заголовок соответствует секции
|
||||
|
||||
alt:
|
||||
- описывает изображение
|
||||
- ключ встроен естественно
|
||||
- нет списка ключей
|
||||
|
||||
filename:
|
||||
- имя файла осмысленное
|
||||
- нет набора ключей через дефисы
|
||||
- имя файла не конфликтует с содержанием изображения
|
||||
```
|
||||
|
||||
## 7. Пороговые значения MVP
|
||||
|
||||
Пороги должны быть настраиваемыми, потому что B2B/технические тексты неизбежно повторяют термины.
|
||||
|
||||
```yaml
|
||||
thresholds:
|
||||
keyword_density:
|
||||
section_warning: 3
|
||||
section_problem: 5
|
||||
exact_keyword:
|
||||
default_limit: 2
|
||||
classic_nausea:
|
||||
warning: 4
|
||||
problem: 7
|
||||
academic_nausea_percent:
|
||||
warning: 7
|
||||
problem: 10
|
||||
proximity:
|
||||
min_sentences_between_same_keyword: 2
|
||||
water_percent:
|
||||
warning: 20
|
||||
problem: 30
|
||||
```
|
||||
|
||||
Пороги не должны автоматически запрещать текст. Они создают предупреждения.
|
||||
|
||||
## 8. Report
|
||||
|
||||
```ts
|
||||
type OverSeoReport = {
|
||||
scope: "page" | "section" | "title" | "description" | "heading" | "alt" | "filename";
|
||||
sourceId: string;
|
||||
sectionId?: string;
|
||||
|
||||
counts: {
|
||||
charsWithSpace: number;
|
||||
charsWithoutSpace: number;
|
||||
words: number;
|
||||
sentences: number;
|
||||
};
|
||||
|
||||
scores: {
|
||||
waterPercent?: number;
|
||||
classicNausea?: number;
|
||||
academicNauseaPercent?: number;
|
||||
maxKeywordDensityPercent?: number;
|
||||
};
|
||||
|
||||
keys: Array<{
|
||||
phrase: string;
|
||||
count: number;
|
||||
densityPercent: number;
|
||||
type?: "primary" | "secondary" | "soft_form" | "detected";
|
||||
sectionAllowed?: boolean;
|
||||
}>;
|
||||
|
||||
keyGroups: Array<{
|
||||
phrase: string;
|
||||
count: number;
|
||||
subKeys: Array<{
|
||||
phrase: string;
|
||||
count: number;
|
||||
}>;
|
||||
}>;
|
||||
|
||||
warnings: Array<{
|
||||
severity: "info" | "warning" | "critical";
|
||||
code: string;
|
||||
message: string;
|
||||
phrase?: string;
|
||||
count?: number;
|
||||
limit?: number;
|
||||
snippet?: string;
|
||||
}>;
|
||||
|
||||
verdict: {
|
||||
status: "ok" | "review" | "problem";
|
||||
summary: string;
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
## 9. UI
|
||||
|
||||
UI должен показывать:
|
||||
|
||||
```text
|
||||
Summary:
|
||||
- status
|
||||
- words count
|
||||
- water score
|
||||
- classic nausea
|
||||
- academic nausea
|
||||
- max keyword density
|
||||
|
||||
Keywords:
|
||||
- phrase
|
||||
- count
|
||||
- density
|
||||
- allowed section
|
||||
- warnings
|
||||
|
||||
Warnings:
|
||||
- severity
|
||||
- reason
|
||||
- snippet
|
||||
- suggested action
|
||||
|
||||
Scopes:
|
||||
- page
|
||||
- sections
|
||||
- title/description
|
||||
- headings
|
||||
- alt
|
||||
- filename
|
||||
```
|
||||
|
||||
## 10. Codex role
|
||||
|
||||
Codex не считает метрики вместо инструмента.
|
||||
|
||||
Codex используется после отчёта:
|
||||
|
||||
```text
|
||||
1. Объяснить предупреждения человеческим языком.
|
||||
2. Предложить варианты исправления.
|
||||
3. Сохранить смысл и тон.
|
||||
4. Не увеличивать текст без необходимости.
|
||||
5. Не вставлять ключи, если отчёт показывает перебор.
|
||||
```
|
||||
|
||||
## 11. Главное ограничение
|
||||
|
||||
`checkOverseo` не должен становиться тупой машиной запретов.
|
||||
|
||||
Он показывает риски:
|
||||
|
||||
```text
|
||||
перебор
|
||||
слишком близко
|
||||
слишком часто
|
||||
слишком водно
|
||||
слишком похоже на набор ключей
|
||||
```
|
||||
|
||||
Финальное решение принимает пользователь.
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
# seo_mode design system notes
|
||||
|
||||
Короткая памятка, чтобы интерфейс не расползался при быстрых итерациях.
|
||||
|
||||
## Визуальный тон
|
||||
|
||||
- База: светлая Apple-like система, много воздуха, мягкие панели, скругления и спокойный серый фон.
|
||||
- Главный акцент: тёмный градиент `#1f1f21 -> #3a332e`, как у активного этапа и основных submit/action-кнопок.
|
||||
- Вторичный акцент: оранжевый `--accent: #f27a1a`, только для выбора папки, drag-and-drop и редких импортных действий.
|
||||
- Избегаем случайной зелени/голубого/фиолета, если это не специально введённый статус.
|
||||
|
||||
## Кнопки
|
||||
|
||||
- Primary actions: `Создать проект`, `Анализировать`, будущие основные шаги этапов. Используют тёмный градиент активного этапа.
|
||||
- Import action: `Выбрать папку`. Остаётся оранжевой, потому что это отдельное действие загрузки/импорта.
|
||||
- Icon actions: копировать, удалить, настройки. Круглые `42x42`, нейтральная светлая поверхность, тонкая граница, мягкая тень.
|
||||
- Danger icon: нейтральная круглая поверхность в покое, красный значок; красная заливка только на hover.
|
||||
|
||||
## Типографика
|
||||
|
||||
- Заголовки секций: плотные, тёмные, без лишних декоративных цветов.
|
||||
- Helper copy под инпутами: `13px`, `font-weight: 400`, приглушённый `--muted`, line-height около `1.38`.
|
||||
- Placeholder: всегда normal/400, приглушённый, без жирности.
|
||||
- Иконки рядом с заголовками секций выравниваем по первой строке заголовка, а не по центру всего блока с описанием.
|
||||
- Не смешивать в одной карточке слишком много жирных уровней: жирный только у заголовка и главного действия.
|
||||
|
||||
## Карточки и сетки
|
||||
|
||||
- Основная ширина страницы: не заужать без причины, держать рабочую ширину около `min(1180px, 100%)`.
|
||||
- Карточки: светлая полупрозрачная поверхность, `var(--radius-lg/xl)`, тонкая линия `var(--line)`, мягкая тень только на крупных контейнерах.
|
||||
- Не вкладывать много одинаковых карточек друг в друга без визуальной причины.
|
||||
- Каждый этап сверху должен быть отдельной логической страницей, а предыдущие настройки на следующих этапах показываем компактно.
|
||||
|
||||
## Рабочее правило
|
||||
|
||||
Перед новой UI-правкой сначала проверяем, нет ли уже подходящего класса/паттерна. Если стиль повторяется второй раз, выносим его в общий паттерн, а не лепим ещё один локальный костыль.
|
||||
|
|
@ -0,0 +1,459 @@
|
|||
# SEO Farm Implementation Guardrails
|
||||
|
||||
## 1. Зачем этот документ
|
||||
|
||||
Этот файл фиксирует порядок разработки и технические ограничения, чтобы user flow, backend pipeline и инструменты не начали спорить друг с другом.
|
||||
|
||||
Главная идея:
|
||||
|
||||
```text
|
||||
UI показывает понятный SEO-процесс.
|
||||
Backend внутри выполняет строгую техническую последовательность.
|
||||
Инструменты не смешивают зоны ответственности.
|
||||
Target project меняется только после ручного Apply.
|
||||
```
|
||||
|
||||
## 2. User Flow vs Backend Pipeline
|
||||
|
||||
Для пользователя первый смысловой шаг после создания проекта — “Проанализировать проект”.
|
||||
|
||||
В UI это один сценарий:
|
||||
|
||||
```text
|
||||
Create Project
|
||||
-> Analyze Project
|
||||
-> Baseline Audit Result
|
||||
```
|
||||
|
||||
Но backend не может делать audit без технической подготовки. Внутренний порядок должен быть таким:
|
||||
|
||||
```text
|
||||
1. validate targetProjectPath
|
||||
2. scan filesystem
|
||||
3. filter ignored folders/files
|
||||
4. detect candidate files
|
||||
5. parse HTML/MD/text/CSS references
|
||||
6. build projectIndex
|
||||
7. map pages
|
||||
8. map sections
|
||||
9. map media assets
|
||||
10. create source mapping
|
||||
11. run baseline audit
|
||||
12. save scan + audit result
|
||||
13. return UI-ready report
|
||||
```
|
||||
|
||||
Для будущих источников backend должен мыслить не только `targetProjectPath`, а `project source`:
|
||||
|
||||
```text
|
||||
MVP:
|
||||
local_folder -> targetProjectPath
|
||||
|
||||
Later:
|
||||
git_repo
|
||||
zip_upload
|
||||
site_crawl
|
||||
```
|
||||
|
||||
Нельзя делать:
|
||||
|
||||
```text
|
||||
Baseline Audit
|
||||
-> потом parse
|
||||
```
|
||||
|
||||
Правильно:
|
||||
|
||||
```text
|
||||
Scan / Parse / Map
|
||||
-> Baseline Audit
|
||||
```
|
||||
|
||||
В UI слово “parse” лучше не выносить отдельным этапом. Это внутренняя операция.
|
||||
|
||||
## 3. Строгий MVP Pipeline
|
||||
|
||||
```text
|
||||
Project Setup
|
||||
input: project name, project source
|
||||
output: project record
|
||||
|
||||
Project Scan
|
||||
input: project source
|
||||
output: projectIndex
|
||||
|
||||
Baseline Audit
|
||||
input: projectIndex
|
||||
output: audit report, page/media/source issues
|
||||
|
||||
Page Workspace
|
||||
input: selected page/group
|
||||
output: originalText, workingText, sectionMap
|
||||
|
||||
Meaning Extraction
|
||||
input: originalText, sectionMap, meta/headings/media context
|
||||
output: section meanings, seed queries, competing intents
|
||||
|
||||
Seed Review
|
||||
input: proposed seeds
|
||||
output: approved seeds
|
||||
|
||||
Wordstat Collection
|
||||
input: approved seeds
|
||||
output: raw Wordstat results, ВЧ/СЧ/НЧ groups
|
||||
|
||||
Keyword Cleaning
|
||||
input: raw Wordstat results
|
||||
output: approved keyword decisions
|
||||
|
||||
Keyword Map
|
||||
input: approved keywords, section meanings, current text
|
||||
output: approved keyword-to-section map
|
||||
|
||||
Optimization Plan
|
||||
input: keyword map, audit, current workingText, design limits
|
||||
output: approved edit plan
|
||||
|
||||
Rewrite Workspace
|
||||
input: approved edit plan
|
||||
output: edited workingText
|
||||
|
||||
Media SEO
|
||||
input: media map, section context, keyword map
|
||||
output: approved alt/title/aria/caption/filename suggestions
|
||||
|
||||
Final Validation
|
||||
input: workingText, keyword map, media map, approved media changes
|
||||
output: validation report
|
||||
|
||||
Apply / Export
|
||||
input: approved text/media changes
|
||||
output: changeset, updated target project, reports/export
|
||||
```
|
||||
|
||||
## 4. Tool Boundaries
|
||||
|
||||
### 4.1. `checkOverseo`
|
||||
|
||||
Отвечает за SEO-перебор:
|
||||
|
||||
```text
|
||||
- точные вхождения;
|
||||
- soft forms;
|
||||
- перетошнота;
|
||||
- заспамленность;
|
||||
- повторы рядом;
|
||||
- повторяющиеся биграммы/триграммы;
|
||||
- перебор ключа в title/description/h1/h2;
|
||||
- keyword stuffing в alt/filename;
|
||||
- конфликт ключа со смыслом секции.
|
||||
```
|
||||
|
||||
Не отвечает за:
|
||||
|
||||
```text
|
||||
- стиль бренда;
|
||||
- красивость текста;
|
||||
- смысловую редактуру;
|
||||
- выбор ключей из Wordstat.
|
||||
```
|
||||
|
||||
### 4.2. `ru-text`
|
||||
|
||||
Отвечает за редакторский фильтр после SEO-правок:
|
||||
|
||||
```text
|
||||
- тон;
|
||||
- читаемость;
|
||||
- канцелярит;
|
||||
- мутные формулировки;
|
||||
- редакторские и типографические проблемы.
|
||||
```
|
||||
|
||||
Не отвечает за:
|
||||
|
||||
```text
|
||||
- сбор ключей;
|
||||
- частотность;
|
||||
- Wordstat;
|
||||
- карту ключей;
|
||||
- SEO-плотность.
|
||||
```
|
||||
|
||||
### 4.3. `keywordService`
|
||||
|
||||
Отвечает за чистку Wordstat-результатов:
|
||||
|
||||
```text
|
||||
- нормализация запросов;
|
||||
- дедупликация;
|
||||
- project blacklist;
|
||||
- мусорные интенты;
|
||||
- группировка близких запросов;
|
||||
- Codex-классификация спорных запросов;
|
||||
- user approval.
|
||||
```
|
||||
|
||||
Это не `stopword`-инструмент и не удаление слов из текста.
|
||||
|
||||
### 4.4. Wordstat MCP
|
||||
|
||||
Wordstat MCP — только “рука в Wordstat”.
|
||||
|
||||
Он:
|
||||
|
||||
```text
|
||||
- получает seed;
|
||||
- возвращает запросы и частотность;
|
||||
- сохраняет raw data.
|
||||
```
|
||||
|
||||
Он не:
|
||||
|
||||
```text
|
||||
- решает, что использовать;
|
||||
- пишет текст;
|
||||
- строит карту ключей;
|
||||
- проверяет переспам.
|
||||
```
|
||||
|
||||
### 4.5. Codex Exec
|
||||
|
||||
Codex через `codex exec` используется для смысловых задач:
|
||||
|
||||
```text
|
||||
- выделить темы;
|
||||
- предложить seed queries;
|
||||
- классифицировать спорные ключи;
|
||||
- предложить keyword map;
|
||||
- сформировать optimization plan;
|
||||
- предложить варианты правки;
|
||||
- предложить alt/caption/filename.
|
||||
```
|
||||
|
||||
Codex не должен напрямую менять target project до `Apply`.
|
||||
|
||||
## 5. Apply Safety Rules
|
||||
|
||||
До ручного Apply target project считается read-only.
|
||||
|
||||
Разрешённые изменения после approval:
|
||||
|
||||
```text
|
||||
- visible text;
|
||||
- title;
|
||||
- description;
|
||||
- headings;
|
||||
- alt/title/aria;
|
||||
- captions;
|
||||
- approved filenames.
|
||||
```
|
||||
|
||||
Запрещено без отдельного подтверждения:
|
||||
|
||||
```text
|
||||
- менять layout;
|
||||
- менять CSS/JS-логику;
|
||||
- рефакторить код сайта;
|
||||
- удалять файлы;
|
||||
- менять структуру приложения SEO Farm;
|
||||
- публиковать на хостинг.
|
||||
```
|
||||
|
||||
Перед Apply UI обязан показать:
|
||||
|
||||
```text
|
||||
- affected files;
|
||||
- text diff;
|
||||
- media diff;
|
||||
- filename rename plan;
|
||||
- validation status;
|
||||
- changeset summary.
|
||||
```
|
||||
|
||||
Apply должен идти транзакционно:
|
||||
|
||||
```text
|
||||
1. build changeset
|
||||
2. run dry-run
|
||||
3. backup affected files to object storage
|
||||
4. apply text/media/filename changes through backend
|
||||
5. verify references and rename results
|
||||
6. rollback if any critical step fails
|
||||
```
|
||||
|
||||
## 5.1. Repeated Scan / Reconcile Rules
|
||||
|
||||
Повторный scan не должен перетирать уже принятые решения.
|
||||
|
||||
Правильно:
|
||||
|
||||
```text
|
||||
new scan
|
||||
-> create scan_version
|
||||
-> create page/section/media versions
|
||||
-> run reconcile
|
||||
-> preserve stable logical IDs
|
||||
-> show review summary to user
|
||||
```
|
||||
|
||||
Стабильные идентификаторы:
|
||||
|
||||
```text
|
||||
page_id
|
||||
section_id
|
||||
media_id
|
||||
```
|
||||
|
||||
Текущий `DOM selector` — это версия привязки, а не identity.
|
||||
|
||||
Статусы reconcile:
|
||||
|
||||
```text
|
||||
matched
|
||||
changed
|
||||
new
|
||||
orphaned
|
||||
conflicted
|
||||
```
|
||||
|
||||
Решения пользователя должны ссылаться на logical IDs:
|
||||
|
||||
```text
|
||||
- keyword decisions
|
||||
- drafts
|
||||
- media approvals
|
||||
- changesets
|
||||
```
|
||||
|
||||
## 6. Storage Risks
|
||||
|
||||
Нельзя хранить всё только в памяти.
|
||||
|
||||
Минимально нужно:
|
||||
|
||||
```text
|
||||
Postgres:
|
||||
- projects
|
||||
- project_sources
|
||||
- scan_versions
|
||||
- pages / page_versions
|
||||
- sections / section_versions
|
||||
- media_assets / media_versions
|
||||
- runs
|
||||
- seeds
|
||||
- wordstat_jobs / wordstat_results
|
||||
- keyword_decisions
|
||||
- keyword_map_items
|
||||
- drafts / draft_items
|
||||
- validations / validation_issues
|
||||
- changesets / changeset_items
|
||||
|
||||
Object Storage:
|
||||
- raw Wordstat dumps
|
||||
- originalText snapshots
|
||||
- workingText drafts
|
||||
- reports
|
||||
- previews / video frames
|
||||
- backups
|
||||
- exports
|
||||
```
|
||||
|
||||
Правило:
|
||||
|
||||
```text
|
||||
Postgres = source of truth
|
||||
Object Storage = heavy artifacts
|
||||
YAML/JSON/Markdown = export/debug only
|
||||
```
|
||||
|
||||
Риск:
|
||||
|
||||
```text
|
||||
Если workingText не отделить от originalText,
|
||||
модельная ошибка может испортить исходник без понятного отката.
|
||||
```
|
||||
|
||||
MVP может хранить одну активную рабочую версию на секцию, но scan versions, snapshots и changesets должны быть полноценными уже с первого релиза.
|
||||
|
||||
## 7. Development Risk Checklist
|
||||
|
||||
### 7.1. Pipeline risks
|
||||
|
||||
- Audit запущен до parse/map.
|
||||
- UI показывает parse как отдельную пользовательскую задачу.
|
||||
- Project scan требует ручного выбора файлов.
|
||||
- Многостраничный сайт превращается в ручной список `index.html`.
|
||||
- Повторный scan затирает уже утверждённые решения без предупреждения.
|
||||
- Решения по секции привязаны только к текущему selector и теряются после перестройки DOM.
|
||||
|
||||
### 7.2. Model risks
|
||||
|
||||
- Codex меняет target project до Apply.
|
||||
- Prompt не ограничивает зону правок.
|
||||
- Model output невалидный JSON, а backend применяет его без проверки.
|
||||
- Модель смешивает SEO-правки с редизайном или рефакторингом кода.
|
||||
|
||||
### 7.3. Keyword risks
|
||||
|
||||
- Wordstat results считаются готовой keyword map.
|
||||
- Keyword Cleaning превращается в ручную таблицу без assisted classification.
|
||||
- ВЧ-запросы насильно пихаются в каждую секцию.
|
||||
- `article`-ключи запихиваются в главную страницу вместо будущих материалов.
|
||||
- Нет user approval перед использованием ключей.
|
||||
|
||||
### 7.4. Text quality risks
|
||||
|
||||
- `checkOverseo` пытается оценивать тон вместо переспама.
|
||||
- `ru-text` используется как SEO-решатель вместо редакторского фильтра.
|
||||
- Проверка длины игнорирует дизайн-блоки.
|
||||
- Правки раздувают текст и ломают визуальную секцию.
|
||||
|
||||
### 7.5. Media SEO risks
|
||||
|
||||
- Alt набивается ключами без связи с изображением.
|
||||
- Decorative media получают SEO-alt.
|
||||
- Видео обрабатывается как `<img alt="">`, хотя у video нет обычного alt.
|
||||
- Filename меняется без обновления ссылок в HTML/CSS/srcset.
|
||||
- Rename plan применяется без diff.
|
||||
|
||||
### 7.6. Apply/export risks
|
||||
|
||||
- Apply происходит без changeset.
|
||||
- Apply происходит без dry-run и backup.
|
||||
- Rename/update references падает посередине без rollback.
|
||||
- Export не включает отчёты.
|
||||
- После apply невозможно понять, какие SEO-решения были приняты.
|
||||
- Автопубликация появляется раньше ручного контроля.
|
||||
|
||||
## 8. Recommended Development Order
|
||||
|
||||
Чтобы не собрать хаос, реализация должна идти так:
|
||||
|
||||
```text
|
||||
1. Project storage + Postgres + Object Storage
|
||||
2. Project setup screen
|
||||
3. project source validation
|
||||
4. project scan
|
||||
5. scan_versions + reconcile
|
||||
6. projectIndex
|
||||
7. baseline audit
|
||||
8. page workspace with originalText/workingText
|
||||
9. CodexExecProvider
|
||||
10. meaning extraction
|
||||
11. seed review
|
||||
12. Wordstat MCP integration
|
||||
13. keywordService
|
||||
14. keyword map
|
||||
15. optimization plan
|
||||
16. rewrite workspace
|
||||
17. media SEO
|
||||
18. checkOverseo
|
||||
19. ru-text
|
||||
20. final validation report
|
||||
21. diff + changeset
|
||||
22. apply with dry-run/backup/rollback
|
||||
23. export/history
|
||||
```
|
||||
|
||||
Нельзя начинать с rewrite UI до появления `originalText/workingText`, иначе легко получить приложение, которое красиво пишет текст, но не умеет безопасно применять изменения.
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
# SEO Farm
|
||||
|
||||
Локальный SEO-редакторский контур для работы с существующими сайтами: scan проекта, baseline audit, Wordstat, keyword map, rewrite workspace, media SEO, final validation и ручной Apply через changeset.
|
||||
|
||||
## Статус
|
||||
|
||||
Архитектурная документация остаётся источником направления, но код приложения уже инициализирован.
|
||||
|
||||
Готов текущий MVP-срез:
|
||||
|
||||
- базовый React/Vite UI;
|
||||
- Node.js backend;
|
||||
- Postgres как source of truth;
|
||||
- MinIO/S3-compatible storage для артефактов;
|
||||
- создание, переименование, копирование и удаление проектов;
|
||||
- импорт папки сайта через browser folder picker / drag-and-drop;
|
||||
- scan project + `project-index.json`;
|
||||
- stable pages/media records и repeated scan;
|
||||
- baseline audit;
|
||||
- выбор страниц для дальнейшей работы деревом по путям;
|
||||
- severity-фильтры и отдельный media issue list;
|
||||
- отображение ошибок только по выбранным страницам;
|
||||
- Page Workspace MVP: preview страницы, ошибки выбранной страницы, логические секции, section editor, save/reset без изменения исходного проекта.
|
||||
|
||||
Текущий рабочий этап: довести `Page Workspace / Draft Model` до кликабельного редакторского сценария.
|
||||
|
||||
## Основные документы
|
||||
|
||||
- `SEO_FARM_BRIEF.md` — общее ТЗ, цели продукта и MVP scope.
|
||||
- `SEO_FARM_ARCHITECTURE.md` — архитектура, storage, backend/frontend слои.
|
||||
- `USER_FLOW.md` — пользовательский сценарий.
|
||||
- `IMPLEMENTATION_GUARDRAILS.md` — pipeline, границы инструментов и риски.
|
||||
- `UI_UX_SPEC.md` — UI/UX экраны, состояния и safety-паттерны.
|
||||
- `TASK_MANAGER.md` — поэтапный план реализации.
|
||||
- `CHECK_OVERSEO_SPEC.md` — спецификация локальной проверки переспама.
|
||||
|
||||
## Зафиксированные архитектурные решения
|
||||
|
||||
- `Postgres` — основной источник истины.
|
||||
- `MinIO/S3` — хранилище raw artifacts, snapshots, reports, backups и exports.
|
||||
- `browser_folder` — текущий рабочий adapter для импорта папки в MinIO и будущего hosted-сценария.
|
||||
- `local_folder` — dev/local adapter для ручного абсолютного пути.
|
||||
- `scan_version + reconcile` — модель повторного скана без потери утверждённых решений.
|
||||
- `Apply` проходит через changeset, dry-run, backup и rollback.
|
||||
|
|
@ -0,0 +1,833 @@
|
|||
# SEO Farm Architecture
|
||||
|
||||
## 1. Общая схема
|
||||
|
||||
```text
|
||||
React/Vite UI
|
||||
|
|
||||
v
|
||||
Local Backend
|
||||
|
|
||||
+-- Project Storage
|
||||
| |
|
||||
| +-- Postgres DB (source of truth)
|
||||
| +-- Object Storage
|
||||
| | +-- MinIO in local/dev
|
||||
| | +-- S3 in hosted mode later
|
||||
| +-- Project Source Adapters
|
||||
| | +-- local_folder (MVP)
|
||||
| | +-- later: git_repo
|
||||
| | +-- later: zip_upload
|
||||
| | +-- later: site_crawl
|
||||
| +-- project_sources
|
||||
| +-- scan_versions
|
||||
| +-- page/section/media logical records
|
||||
| +-- page_versions / section_versions / media_versions
|
||||
| +-- originalText snapshots
|
||||
| +-- workingText drafts
|
||||
| +-- run history
|
||||
| +-- changesets
|
||||
| +-- backups
|
||||
| +-- exports
|
||||
| +-- reports/raw dumps/previews/frames
|
||||
|
|
||||
+-- Task Runner
|
||||
|
|
||||
+-- Model Provider
|
||||
| |
|
||||
| +-- MVP: Codex Exec Provider
|
||||
| | |
|
||||
| | +-- codex exec
|
||||
| |
|
||||
| +-- Later: OpenAI API Provider
|
||||
| +-- Later: Other API Provider
|
||||
| +-- Later: Local LLM Provider
|
||||
|
|
||||
+-- SEO Tools
|
||||
| |
|
||||
| +-- HTML/Text Extractor
|
||||
| +-- Structure Checker
|
||||
| +-- Keyword Checker
|
||||
| +-- Over-SEO Checker
|
||||
| +-- Page Map Checker
|
||||
|
|
||||
+-- Wordstat Layer
|
||||
| |
|
||||
| +-- MCP-KV Wordstat
|
||||
|
|
||||
+-- Text Quality Layer
|
||||
| |
|
||||
| +-- ru-text
|
||||
|
|
||||
+-- Media SEO Layer
|
||||
|
|
||||
+-- HTML media parser
|
||||
+-- optional: image-size
|
||||
+-- optional: Sharp
|
||||
+-- optional: FFmpeg
|
||||
+-- Model Provider for image/video description
|
||||
```
|
||||
|
||||
## 1.1. Статус инструментов
|
||||
|
||||
```text
|
||||
MVP, точно используем:
|
||||
- React
|
||||
- Vite
|
||||
- Node.js backend
|
||||
- codex exec
|
||||
- ModelProvider abstraction
|
||||
- MCP-KV Wordstat
|
||||
- ru-text
|
||||
- Cheerio
|
||||
- local SEO checks
|
||||
- Postgres
|
||||
- MinIO or other S3-compatible object storage
|
||||
- structured JSON model outputs
|
||||
- YAML/JSON/Markdown exports for debug/import-export only
|
||||
|
||||
Опционально для Media SEO, подключать когда дойдём до медиа:
|
||||
- image-size
|
||||
- Sharp
|
||||
- FFmpeg
|
||||
|
||||
Не используем в MVP:
|
||||
- автопостинг
|
||||
- Яндекс Метрика
|
||||
- robots/sitemap editor
|
||||
- hosted/cloud mode
|
||||
- Codex skills as dependency
|
||||
```
|
||||
|
||||
Future enhancement:
|
||||
|
||||
```text
|
||||
Codex skills can be added later to improve repeatable model workflows:
|
||||
- seo-seed-extractor
|
||||
- seo-keyword-classifier
|
||||
- seo-copy-editor-ru
|
||||
- media-seo-alt-writer
|
||||
|
||||
Skills are not required for MVP. First stabilize tools and reports.
|
||||
```
|
||||
|
||||
## 1.2. Project Source Adapters
|
||||
|
||||
```text
|
||||
SourceAdapter
|
||||
|
|
||||
+-- local_folder
|
||||
| +-- MVP source type
|
||||
| +-- uses targetProjectPath
|
||||
|
|
||||
+-- git_repo
|
||||
| +-- future hosted/local source
|
||||
|
|
||||
+-- zip_upload
|
||||
| +-- future hosted/local source
|
||||
|
|
||||
+-- site_crawl
|
||||
+-- future audit-only source
|
||||
```
|
||||
|
||||
For MVP we start with `local_folder`, but the application should keep the source type in the database from day one so local and hosted modes use the same upper pipeline.
|
||||
|
||||
## 2. Основной поток данных
|
||||
|
||||
```text
|
||||
1. User creates/selects SEO Farm project
|
||||
|
|
||||
v
|
||||
2. User sets project source for the tested site/codebase
|
||||
|
|
||||
+-- MVP: local_folder -> targetProjectPath
|
||||
|
|
||||
v
|
||||
3. Backend creates scan_version, scans source, and builds projectIndex
|
||||
|
|
||||
v
|
||||
4. Baseline Audit runs on discovered pages/assets/media
|
||||
|
|
||||
v
|
||||
5. User selects page or page group for SEO work
|
||||
|
|
||||
v
|
||||
6. Backend creates originalText snapshots and workingText drafts
|
||||
|
|
||||
v
|
||||
7. Model Provider extracts meanings and seed queries
|
||||
|
|
||||
v
|
||||
8. User reviews/edits seed queries
|
||||
|
|
||||
v
|
||||
9. Wordstat Layer collects keyword data
|
||||
|
|
||||
v
|
||||
10. keywordService cleans/classifies keyword results
|
||||
|
|
||||
v
|
||||
11. User reviews keyword decisions
|
||||
|
|
||||
v
|
||||
12. Keyword Map Builder proposes section mapping and limits
|
||||
|
|
||||
v
|
||||
13. User approves/edits Keyword Map
|
||||
|
|
||||
v
|
||||
14. Optimization Plan is generated and approved
|
||||
|
|
||||
v
|
||||
15. Model Provider proposes text edits into workingText
|
||||
|
|
||||
v
|
||||
16. Media SEO proposes alt/caption/filename edits
|
||||
|
|
||||
v
|
||||
17. SEO Tools + ru-text validate text and media edits
|
||||
|
|
||||
v
|
||||
18. UI shows report and diff
|
||||
|
|
||||
v
|
||||
19. User applies changes to target project
|
||||
|
|
||||
v
|
||||
20. User can export archive/reports
|
||||
```
|
||||
|
||||
MVP works with existing code/text only:
|
||||
|
||||
```text
|
||||
existing HTML / existing draft / existing site
|
||||
-> analyze
|
||||
-> collect keywords
|
||||
-> edit
|
||||
-> final validate
|
||||
```
|
||||
|
||||
Creating text from scratch is a future mode, not part of MVP.
|
||||
|
||||
## 3. UI Modules
|
||||
|
||||
```text
|
||||
app/
|
||||
Projects
|
||||
-> create/select local project
|
||||
-> set target project path
|
||||
-> scan project
|
||||
|
||||
Project Dashboard
|
||||
-> project status
|
||||
-> discovered pages
|
||||
-> baseline audit status
|
||||
-> recent runs
|
||||
-> working drafts
|
||||
-> changesets
|
||||
-> exports
|
||||
|
||||
Baseline Audit
|
||||
-> project/page structure
|
||||
-> title/description/h1/h2
|
||||
-> discovered text blocks
|
||||
-> media issues
|
||||
-> technical SEO issues
|
||||
|
||||
Pages / Source
|
||||
-> discovered pages
|
||||
-> selected page
|
||||
-> extracted sections
|
||||
-> extracted headings
|
||||
-> original text
|
||||
-> working text
|
||||
|
||||
Seeds
|
||||
-> seed queries from source text
|
||||
-> manual seed editing
|
||||
-> select seeds for Wordstat
|
||||
|
||||
Wordstat
|
||||
-> run MCP-KV Wordstat
|
||||
-> show frequency/dynamics/regions
|
||||
-> raw keyword results
|
||||
-> high/mid/low groups
|
||||
|
||||
Keyword Cleaning
|
||||
-> normalize/deduplicate Wordstat queries
|
||||
-> mark obvious trash
|
||||
-> classify ambiguous queries through Codex
|
||||
-> approve/edit statuses: use/support/article/risky/trash
|
||||
|
||||
Keyword Map
|
||||
-> suggested keyword-to-section map
|
||||
-> approve/edit suggestions
|
||||
-> primary/secondary keywords
|
||||
-> soft forms
|
||||
-> exact-match limits
|
||||
|
||||
Optimization Plan
|
||||
-> sections to edit
|
||||
-> sections to leave unchanged
|
||||
-> keywords to add/reduce
|
||||
-> media tasks
|
||||
-> length/tone risks
|
||||
|
||||
Rewrite
|
||||
-> select section
|
||||
-> edit working text
|
||||
-> generate variants through Model Provider
|
||||
-> run SEO checks
|
||||
-> run ru-text
|
||||
-> show original vs working diff
|
||||
-> apply only after user approval
|
||||
|
||||
Media SEO
|
||||
-> list images/videos/posters
|
||||
-> current alt/title/aria/captions
|
||||
-> preview
|
||||
-> suggested alt/descriptions
|
||||
-> media SEO warnings
|
||||
|
||||
Final Validation
|
||||
-> title/description/h1/h2
|
||||
-> competing meanings
|
||||
-> keyword usage
|
||||
-> over-SEO warnings
|
||||
-> text length/design limits
|
||||
-> ru-text status
|
||||
-> media SEO status
|
||||
|
||||
Apply / Export
|
||||
-> affected files
|
||||
-> changeset
|
||||
-> manual approval
|
||||
-> apply to target project
|
||||
-> zip target project
|
||||
-> export reports
|
||||
|
||||
History / Reports
|
||||
-> scans
|
||||
-> audits
|
||||
-> Wordstat runs
|
||||
-> validation reports
|
||||
-> media SEO reports
|
||||
-> keyword decisions
|
||||
-> rewrites
|
||||
-> changesets
|
||||
-> exports
|
||||
```
|
||||
|
||||
## 4. Backend Modules
|
||||
|
||||
```text
|
||||
server/
|
||||
routes/
|
||||
projects
|
||||
scan
|
||||
pages
|
||||
sources
|
||||
seeds
|
||||
wordstat
|
||||
keywords
|
||||
keyword-cleaning
|
||||
keyword-map
|
||||
optimization-plan
|
||||
audit
|
||||
rewrite
|
||||
media
|
||||
validate
|
||||
apply
|
||||
export
|
||||
history
|
||||
reports
|
||||
|
||||
providers/model/
|
||||
ModelProvider.ts
|
||||
CodexExecProvider.ts
|
||||
OpenAiApiProvider.ts # future
|
||||
CustomHttpProvider.ts # future
|
||||
|
||||
providers/storage/
|
||||
StorageProvider.ts
|
||||
MinioStorageProvider.ts
|
||||
|
||||
services/
|
||||
db
|
||||
projectSourceService
|
||||
storageProvider
|
||||
taskRunner
|
||||
projectStorage
|
||||
projectScanService
|
||||
reconcileService
|
||||
pageWorkspaceService
|
||||
wordstatService
|
||||
keywordService
|
||||
keywordMapService
|
||||
optimizationPlanService
|
||||
seoAuditService
|
||||
ruTextService
|
||||
mediaAuditService
|
||||
validationService
|
||||
changesetService
|
||||
applyService
|
||||
exportService
|
||||
reportService
|
||||
```
|
||||
|
||||
## 5. Tool Responsibilities
|
||||
|
||||
| Layer | Tool | Responsibility |
|
||||
|---|---|---|
|
||||
| UI | React | Interface, state, workflows |
|
||||
| UI | Vite | Dev server and build |
|
||||
| Backend | Node.js | Local server and task orchestration |
|
||||
| Backend | Express/Fastify | HTTP API for UI |
|
||||
| Storage | Postgres | Source of truth for projects, sources, versions, approvals, runs, validations, changesets |
|
||||
| Storage | MinIO/S3 | Raw dumps, previews, reports, exports, backups, scan snapshots |
|
||||
| Model | `codex exec` | MVP model execution |
|
||||
| Model | ModelProvider | Switch between Codex/API/local models |
|
||||
| Wordstat | MCP-KV Wordstat | Raw keyword data from Yandex Wordstat |
|
||||
| Text parsing | Cheerio | Parse HTML and extract title, meta, headings, text blocks, media tags |
|
||||
| SEO checks | Local scripts | Title, description, h1/h2, keywords, spam, length |
|
||||
| Text quality | ru-text | Editorial SEO correction, tone, readability, typographic checks |
|
||||
| Media parsing | Cheerio | Find img/picture/video/source/poster |
|
||||
| Images | image-size, optional | Read image dimensions for media report |
|
||||
| Images | Sharp, optional | Generate previews/thumbnails if browser preview is not enough |
|
||||
| Video | FFmpeg, optional | Extract representative frames from videos |
|
||||
| Files | YAML/JSON/Markdown | Export, debug, import/export artifacts only; not the main storage layer |
|
||||
|
||||
## 6. Model Provider Abstraction
|
||||
|
||||
```text
|
||||
ModelProvider
|
||||
|
|
||||
+-- CodexExecProvider
|
||||
| +-- calls: codex exec
|
||||
| +-- use case: local MVP
|
||||
|
|
||||
+-- OpenAiApiProvider
|
||||
| +-- calls: OpenAI API
|
||||
| +-- use case: remote app / hosted version
|
||||
|
|
||||
+-- CustomHttpProvider
|
||||
| +-- calls: any compatible API
|
||||
| +-- use case: future integrations
|
||||
|
|
||||
+-- LocalLlmProvider
|
||||
+-- calls: local model runtime
|
||||
+-- use case: offline/private mode
|
||||
```
|
||||
|
||||
## 7. Wordstat Flow
|
||||
|
||||
```text
|
||||
Seed queries
|
||||
|
|
||||
v
|
||||
wordstatService
|
||||
|
|
||||
v
|
||||
MCP-KV Wordstat
|
||||
|
|
||||
+-- top requests
|
||||
+-- related requests
|
||||
+-- dynamics
|
||||
+-- regions
|
||||
|
|
||||
v
|
||||
raw keyword dump
|
||||
|
|
||||
v
|
||||
keywordService
|
||||
|
|
||||
+-- clean by local rules
|
||||
+-- classify
|
||||
+-- frequency type: high/mid/low
|
||||
+-- status: use/support/article/risky/trash
|
||||
```
|
||||
|
||||
## 7.1. Keyword Cleaning Logic
|
||||
|
||||
Чистка мусора — это не stopword/удаление частых слов. Это смысловая фильтрация Wordstat-ключей на релевантность проекту. Делает её `keywordService`: локальные правила проекта плюс Codex-классификация спорных запросов.
|
||||
|
||||
```text
|
||||
Raw Wordstat keywords
|
||||
|
|
||||
v
|
||||
keywordService
|
||||
|
|
||||
+-- normalize text
|
||||
+-- remove duplicates
|
||||
+-- apply negative intents
|
||||
+-- apply project blacklist
|
||||
+-- group close phrases
|
||||
+-- mark obvious trash
|
||||
+-- ask Model Provider to classify ambiguous phrases
|
||||
|
|
||||
v
|
||||
User review
|
||||
```
|
||||
|
||||
Базовые причины выкинуть запрос:
|
||||
|
||||
```text
|
||||
- вакансии
|
||||
- обучение / курсы
|
||||
- скачать
|
||||
- бесплатно
|
||||
- реферат / диплом
|
||||
- бытовой смысл вместо B2B
|
||||
- чужая отрасль
|
||||
- слишком широкий запрос без связи с продуктом
|
||||
- запрос конфликтует с primary_intent страницы
|
||||
```
|
||||
|
||||
Результат хранится как статус:
|
||||
|
||||
```text
|
||||
use / support / article / risky / trash
|
||||
```
|
||||
|
||||
## 8. Text SEO Flow
|
||||
|
||||
```text
|
||||
Source text / HTML
|
||||
|
|
||||
v
|
||||
extractText
|
||||
|
|
||||
+-- sections
|
||||
+-- headings
|
||||
+-- visible text
|
||||
|
|
||||
v
|
||||
Model Provider
|
||||
|
|
||||
+-- meanings
|
||||
+-- seed queries
|
||||
|
|
||||
v
|
||||
SEO audit
|
||||
|
|
||||
+-- title/description/h1/h2
|
||||
+-- competing meanings
|
||||
+-- keyword usage
|
||||
+-- over-SEO
|
||||
+-- length/design limits
|
||||
|
|
||||
v
|
||||
Rewrite variants
|
||||
|
|
||||
v
|
||||
ru-text + SEO checks
|
||||
|
|
||||
v
|
||||
User approval
|
||||
```
|
||||
|
||||
## 8.1. Over-SEO Checker
|
||||
|
||||
Проверка перетошноты/переблева ключами описана отдельно в `CHECK_OVERSEO_SPEC.md`.
|
||||
|
||||
Text.ru используется только как референс по смыслу показателей. Их API не подключаем.
|
||||
|
||||
`seo-analyzer` и похожие HTML-аудиторы не заменяют этот слой: они полезны для title/h1/alt/структуры, но не решают именно перетошноту текста.
|
||||
|
||||
```text
|
||||
checkOverseo
|
||||
|
|
||||
+-- exact keyword count
|
||||
+-- soft_forms count from keywords.yml
|
||||
+-- keyword density by section
|
||||
+-- classic nausea: sqrt(max word/phrase frequency)
|
||||
+-- academic nausea: top word/phrase share in text
|
||||
+-- repeated exact phrases
|
||||
+-- repeated bigrams/trigrams
|
||||
+-- proximity: same keyword too close
|
||||
+-- title/description/h1 overuse
|
||||
+-- alt keyword stuffing
|
||||
+-- section-level warnings
|
||||
```
|
||||
|
||||
Источник правил:
|
||||
|
||||
```text
|
||||
keywords.yml
|
||||
- phrase
|
||||
- soft_forms
|
||||
- section
|
||||
- exact_limit
|
||||
- max_section_count
|
||||
- status
|
||||
- notes
|
||||
|
||||
page-map.yml
|
||||
- primary_intent
|
||||
- section meanings
|
||||
- forbidden_intents
|
||||
- max_lines_hint
|
||||
```
|
||||
|
||||
В MVP не нужен сложный морфологический движок. Для русского текста сначала используем:
|
||||
|
||||
```text
|
||||
- точные ключи;
|
||||
- вручную/моделью заданные soft_forms;
|
||||
- нормализацию регистра и пробелов;
|
||||
- подсчёт повторов по секциям.
|
||||
```
|
||||
|
||||
Если этого станет мало, позже можно добавить отдельный морфологический модуль, но он не входит в MVP.
|
||||
|
||||
## 9. Media SEO Flow
|
||||
|
||||
```text
|
||||
Source HTML/CSS
|
||||
|
|
||||
v
|
||||
extractMedia
|
||||
|
|
||||
+-- img
|
||||
+-- picture/source/srcset
|
||||
+-- video/source/poster
|
||||
+-- CSS backgrounds
|
||||
|
|
||||
v
|
||||
mediaAuditService
|
||||
|
|
||||
+-- current alt/title/aria
|
||||
+-- filename
|
||||
+-- captions
|
||||
+-- section context
|
||||
+-- dimensions
|
||||
+-- file size
|
||||
|
|
||||
+-- optional image-size
|
||||
+-- optional Sharp previews
|
||||
+-- optional FFmpeg video frames
|
||||
|
|
||||
v
|
||||
Model Provider
|
||||
|
|
||||
+-- describe image/video frame
|
||||
+-- suggest alt/caption/description
|
||||
|
|
||||
v
|
||||
SEO checks
|
||||
|
|
||||
+-- missing alt
|
||||
+-- weak alt
|
||||
+-- duplicate alt
|
||||
+-- decorative media
|
||||
+-- weak/non-descriptive filename
|
||||
+-- keyword stuffing in alt
|
||||
+-- missing video context
|
||||
|
|
||||
v
|
||||
User approval
|
||||
```
|
||||
|
||||
## 10. Storage Layout
|
||||
|
||||
```text
|
||||
Postgres
|
||||
- projects
|
||||
- project_sources
|
||||
- scan_versions
|
||||
- pages
|
||||
- page_versions
|
||||
- sections
|
||||
- section_versions
|
||||
- media_assets
|
||||
- media_versions
|
||||
- runs
|
||||
- seeds
|
||||
- wordstat_jobs
|
||||
- wordstat_results
|
||||
- keyword_decisions
|
||||
- keyword_map_items
|
||||
- optimization_plans
|
||||
- drafts
|
||||
- draft_items
|
||||
- validations
|
||||
- validation_issues
|
||||
- changesets
|
||||
- changeset_items
|
||||
- exports
|
||||
|
||||
Object Storage
|
||||
- projects/project-id/scans/scan-version-id/project-index.json
|
||||
- projects/project-id/raw/wordstat/*.json
|
||||
- projects/project-id/snapshots/original/*.json
|
||||
- projects/project-id/snapshots/working/*.json
|
||||
- projects/project-id/reports/*.md
|
||||
- projects/project-id/previews/*
|
||||
- projects/project-id/video-frames/*
|
||||
- projects/project-id/backups/changeset-id/*
|
||||
- projects/project-id/exports/*
|
||||
```
|
||||
|
||||
`project.json` can exist as export/debug metadata, but the source of truth lives in Postgres:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "nodedc",
|
||||
"name": "Node.DC",
|
||||
"source": {
|
||||
"type": "local_folder",
|
||||
"targetProjectPath": "C:/Users/const/Desktop/landing-correction"
|
||||
},
|
||||
"scan": {
|
||||
"include": ["**/*.html", "**/*.md", "css/**/*.css", "images/**/*", "video/**/*"],
|
||||
"exclude": ["node_modules/**", "dist/**", ".git/**"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Postgres stores project state and history:
|
||||
|
||||
```text
|
||||
projects
|
||||
project_sources
|
||||
scan_versions
|
||||
pages
|
||||
page_versions
|
||||
sections
|
||||
section_versions
|
||||
media_assets
|
||||
media_versions
|
||||
runs
|
||||
seeds
|
||||
wordstat_jobs
|
||||
wordstat_results
|
||||
keyword_decisions
|
||||
keyword_map_items
|
||||
optimization_plans
|
||||
drafts
|
||||
draft_items
|
||||
validations
|
||||
validation_issues
|
||||
changesets
|
||||
changeset_items
|
||||
exports
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
```text
|
||||
Postgres = source of truth
|
||||
Object Storage = heavy/snapshot/report artifacts
|
||||
YAML/JSON/Markdown = export/debug/import-export only
|
||||
```
|
||||
|
||||
## 10.1. Repeated Scan / Reconcile Model
|
||||
|
||||
Every scan creates a new `scan_version`.
|
||||
|
||||
Important rule:
|
||||
|
||||
```text
|
||||
DOM selector != entity identity
|
||||
```
|
||||
|
||||
Stable identities:
|
||||
|
||||
```text
|
||||
page_id
|
||||
section_id
|
||||
media_id
|
||||
```
|
||||
|
||||
Versioned snapshots:
|
||||
|
||||
```text
|
||||
page_version
|
||||
section_version
|
||||
media_version
|
||||
```
|
||||
|
||||
When a new scan is created, backend runs reconcile:
|
||||
|
||||
```text
|
||||
matched
|
||||
changed
|
||||
new
|
||||
orphaned
|
||||
conflicted
|
||||
```
|
||||
|
||||
Matching should use more than one signal:
|
||||
|
||||
```text
|
||||
- source path/url
|
||||
- heading/context
|
||||
- normalized text hash
|
||||
- selector fingerprint
|
||||
- neighboring structure
|
||||
```
|
||||
|
||||
Approvals, keyword decisions, drafts, and media decisions must link to stable logical IDs, not only to the current selector.
|
||||
|
||||
Text editing model:
|
||||
|
||||
```text
|
||||
target project file
|
||||
|
|
||||
v
|
||||
extract text
|
||||
|
|
||||
+-- originalText snapshot
|
||||
|
|
||||
+-- workingText draft
|
||||
|
|
||||
+-- user edits
|
||||
+-- Codex edits
|
||||
+-- SEO checks
|
||||
+-- ru-text checks
|
||||
|
|
||||
v
|
||||
diff
|
||||
|
|
||||
v
|
||||
user approval
|
||||
|
|
||||
v
|
||||
changeset
|
||||
|
|
||||
+-- dry-run
|
||||
+-- backup affected files to object storage
|
||||
+-- apply through backend
|
||||
+-- verify references/renames
|
||||
+-- rollback on failure
|
||||
|
|
||||
v
|
||||
apply to target project
|
||||
```
|
||||
|
||||
MVP stores one `workingText` draft per source/section. Full draft versioning and rollback history are future enhancements.
|
||||
|
||||
## 11. External Links
|
||||
|
||||
Related local docs:
|
||||
|
||||
- `SEO_FARM_BRIEF.md`
|
||||
- `USER_FLOW.md`
|
||||
- `CHECK_OVERSEO_SPEC.md`
|
||||
- `IMPLEMENTATION_GUARDRAILS.md`
|
||||
- `UI_UX_SPEC.md`
|
||||
- `TASK_MANAGER.md`
|
||||
|
||||
- Codex exec: https://developers.openai.com/codex/noninteractive
|
||||
- Codex with ChatGPT plan: https://help.openai.com/en/articles/11369540-using-codex-with-your-chatgpt-plan
|
||||
- React: https://react.dev/
|
||||
- Vite: https://vite.dev/guide/
|
||||
- MCP-KV Wordstat: https://mcp-kv.ru/docs/wordstat-mcp-setup
|
||||
- MCP-KV SEO tools, optional/reference: https://mcp-kv.ru/mcp-server-seo
|
||||
- ru-text: https://github.com/talkstream/ru-text
|
||||
- Cheerio: https://cheerio.js.org/docs/api/
|
||||
- image-size: https://github.com/image-size/image-size
|
||||
- Sharp: https://www.npmjs.com/package/sharp
|
||||
- FFmpeg: https://ffmpeg.org/ffmpeg-doc.html
|
||||
- Text.ru SEO analysis reference: https://text.ru/seo
|
||||
- Text.ru API/check description, reference for analysis model only: https://text.ru/api-check
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,642 @@
|
|||
# SEO Farm User Flow
|
||||
|
||||
## 1. Главный принцип
|
||||
|
||||
SEO Farm работает как проектный сервис, а не как одноразовая форма.
|
||||
|
||||
```text
|
||||
Пользователь создаёт проект
|
||||
-> приложение сканирует сайт
|
||||
-> делает baseline audit
|
||||
-> пользователь выбирает страницы
|
||||
-> система помогает собрать ключи
|
||||
-> система предлагает карту ключей
|
||||
-> пользователь правит тексты в working copy
|
||||
-> checkOverseo ловит переспам
|
||||
-> ru-text и редакторские проверки ловят плохой тон
|
||||
-> пользователь применяет изменения
|
||||
-> история работы сохраняется
|
||||
```
|
||||
|
||||
MVP работает с уже существующим кодом/страницей/текстом. Создание текстов с нуля — отдельный будущий режим.
|
||||
|
||||
## 2. Project Lifecycle
|
||||
|
||||
### 2.1. Create Project
|
||||
|
||||
Пользователь создаёт проект и выбирает папку сайта:
|
||||
|
||||
```text
|
||||
source.type = local_folder
|
||||
targetProjectPath = C:/path/to/site
|
||||
```
|
||||
|
||||
Пользователь не выбирает вручную `index.html`, `css`, `images` и другие файлы. Приложение само сканирует проект.
|
||||
|
||||
На этом шаге сохраняется:
|
||||
|
||||
```text
|
||||
project
|
||||
- id
|
||||
- name
|
||||
- source.type
|
||||
- targetProjectPath
|
||||
- createdAt
|
||||
- updatedAt
|
||||
- lastRunAt
|
||||
```
|
||||
|
||||
### 2.2. Project Scan
|
||||
|
||||
Приложение сканирует папку проекта и строит `projectIndex`.
|
||||
|
||||
Для пользователя это часть одного действия “Проанализировать проект”. Внутри backend обязан соблюдать порядок:
|
||||
|
||||
```text
|
||||
scan filesystem
|
||||
-> parse candidate files
|
||||
-> create scan_version
|
||||
-> build projectIndex
|
||||
-> map pages/sections/media
|
||||
-> reconcile with previous scan_version
|
||||
-> run baseline audit
|
||||
-> show audit result in UI
|
||||
```
|
||||
|
||||
Baseline audit не может выполняться до parse/map, но пользователь не должен отдельно управлять техническим парсингом.
|
||||
|
||||
`projectIndex` содержит:
|
||||
|
||||
```text
|
||||
projectIndex
|
||||
- pages
|
||||
- html files
|
||||
- markdown/text files
|
||||
- css files
|
||||
- images
|
||||
- videos
|
||||
- possible entry points
|
||||
- ignored files
|
||||
```
|
||||
|
||||
Repeated scan не должен молча перетирать старые решения. Новый scan обязан:
|
||||
|
||||
```text
|
||||
- создать новый scan_version
|
||||
- попытаться сматчить page/section/media с прошлой версией
|
||||
- сохранить stable IDs
|
||||
- показать reconcile summary: matched/changed/new/orphaned/conflicted
|
||||
```
|
||||
|
||||
Пользователь видит найденные страницы и может:
|
||||
|
||||
- выбрать страницу для работы;
|
||||
- исключить технические/лишние файлы;
|
||||
- переименовать страницу в UI;
|
||||
- запустить повторный scan.
|
||||
|
||||
### 2.3. Project Dashboard
|
||||
|
||||
После scan пользователь попадает в dashboard проекта:
|
||||
|
||||
```text
|
||||
Dashboard
|
||||
- список страниц
|
||||
- статус baseline audit
|
||||
- последние SEO runs
|
||||
- незавершённые working drafts
|
||||
- последние changesets
|
||||
- exports
|
||||
```
|
||||
|
||||
## 3. Baseline Audit
|
||||
|
||||
Baseline audit — первый смысловой шаг пользователя после создания проекта.
|
||||
|
||||
Технически audit использует parse/scan, но в UI пользователь видит именно “Аудит проекта/страницы”.
|
||||
|
||||
Проверяется:
|
||||
|
||||
```text
|
||||
page structure:
|
||||
- title
|
||||
- description
|
||||
- h1/h2
|
||||
- heading order
|
||||
|
||||
content:
|
||||
- видимый текст
|
||||
- секции
|
||||
- desktop/adaptive дубли
|
||||
- потенциальные смысловые блоки
|
||||
|
||||
media:
|
||||
- img/picture/srcset
|
||||
- video/poster/source
|
||||
- alt/title/aria
|
||||
- filename
|
||||
|
||||
technical:
|
||||
- missing meta
|
||||
- empty headings
|
||||
- empty alt
|
||||
- duplicate alt
|
||||
- obvious weak filenames
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
```text
|
||||
baselineAudit
|
||||
- project-level issues
|
||||
- page-level issues
|
||||
- media issues
|
||||
- source mapping
|
||||
```
|
||||
|
||||
## 4. Page Workspace
|
||||
|
||||
Пользователь выбирает страницу или текстовый источник из проекта.
|
||||
|
||||
Для страницы создаются:
|
||||
|
||||
```text
|
||||
originalText
|
||||
- неизменённый снимок текста из target project
|
||||
|
||||
workingText
|
||||
- рабочая копия для SEO-правок
|
||||
|
||||
sectionMap
|
||||
- sectionId
|
||||
- DOM selector
|
||||
- source file
|
||||
- original text
|
||||
- working text
|
||||
- context
|
||||
```
|
||||
|
||||
Codex и пользователь работают только с `workingText`.
|
||||
|
||||
Target project не изменяется до ручного `Apply`.
|
||||
|
||||
## 5. Meaning Extraction
|
||||
|
||||
Codex через `codex exec` анализирует выбранную страницу:
|
||||
|
||||
```text
|
||||
input:
|
||||
- originalText
|
||||
- sectionMap
|
||||
- title/description/h1/h2
|
||||
- media context
|
||||
|
||||
output:
|
||||
- primary_intent
|
||||
- section meanings
|
||||
- product entities
|
||||
- likely seed queries
|
||||
- possible competing intents
|
||||
```
|
||||
|
||||
Пользователь видит смысловую карту и может поправить:
|
||||
|
||||
- главный смысл;
|
||||
- смысл секции;
|
||||
- неверно выделенные сущности;
|
||||
- лишние seed-запросы.
|
||||
|
||||
## 6. Seed Review
|
||||
|
||||
UI показывает seed-запросы:
|
||||
|
||||
```text
|
||||
seed
|
||||
- phrase
|
||||
- source section
|
||||
- reason
|
||||
- confidence
|
||||
- selected true/false
|
||||
```
|
||||
|
||||
Пользователь:
|
||||
|
||||
- включает/выключает seeds;
|
||||
- добавляет свои;
|
||||
- объединяет похожие;
|
||||
- помечает приоритет.
|
||||
|
||||
После утверждения seeds идут в Wordstat.
|
||||
|
||||
## 7. Wordstat Collection
|
||||
|
||||
Wordstat MCP — рука в Wordstat.
|
||||
|
||||
Для выбранных seeds собираются:
|
||||
|
||||
```text
|
||||
wordstat result
|
||||
- seed phrase
|
||||
- related query
|
||||
- frequency
|
||||
- frequency group: high / mid / low
|
||||
- dynamics
|
||||
- region if available
|
||||
```
|
||||
|
||||
UI должен позволять:
|
||||
|
||||
- запускать сбор по одному seed;
|
||||
- запускать batch;
|
||||
- видеть raw results;
|
||||
- повторять сбор;
|
||||
- сохранять результаты в истории проекта.
|
||||
|
||||
## 8. Keyword Cleaning
|
||||
|
||||
Это assisted workflow, не ручная таблица с нуля.
|
||||
|
||||
`keywordService` делает:
|
||||
|
||||
```text
|
||||
1. normalize
|
||||
2. remove duplicates
|
||||
3. apply project blacklist
|
||||
4. mark obvious trash
|
||||
5. group close phrases
|
||||
6. send ambiguous phrases to Codex classification
|
||||
7. return suggested statuses
|
||||
```
|
||||
|
||||
Статусы:
|
||||
|
||||
```text
|
||||
use
|
||||
support
|
||||
article
|
||||
risky
|
||||
trash
|
||||
```
|
||||
|
||||
UI:
|
||||
|
||||
```text
|
||||
Keyword Cleaning screen
|
||||
- таблица ключей
|
||||
- частотность
|
||||
- seed source
|
||||
- proposed status
|
||||
- reason
|
||||
- confidence
|
||||
- bulk actions
|
||||
- filters: high/mid/low/use/trash/risky
|
||||
```
|
||||
|
||||
Пользователь утверждает результат галками и массовыми действиями.
|
||||
|
||||
## 9. Keyword Map
|
||||
|
||||
Keyword Map не должен быть ручной дрочней.
|
||||
|
||||
Система предлагает карту:
|
||||
|
||||
```text
|
||||
keyword -> section
|
||||
keyword -> role
|
||||
keyword -> limit
|
||||
keyword -> soft_forms
|
||||
```
|
||||
|
||||
Codex использует:
|
||||
|
||||
- section meanings;
|
||||
- primary_intent;
|
||||
- keyword statuses;
|
||||
- частотность;
|
||||
- текущий текст секции;
|
||||
- риск переспама.
|
||||
|
||||
Роли:
|
||||
|
||||
```text
|
||||
primary
|
||||
secondary
|
||||
supporting
|
||||
do_not_use_on_page
|
||||
article_backlog
|
||||
```
|
||||
|
||||
UI:
|
||||
|
||||
```text
|
||||
Keyword Map screen
|
||||
- секции страницы слева
|
||||
- предложенные ключи справа
|
||||
- primary/secondary badges
|
||||
- soft_forms
|
||||
- exact_limit
|
||||
- warnings
|
||||
- approve/apply suggestions
|
||||
```
|
||||
|
||||
Пользователь редактирует предложения, но не раскладывает всё с нуля.
|
||||
|
||||
## 10. Optimization Plan
|
||||
|
||||
Перед переписыванием система собирает план:
|
||||
|
||||
```text
|
||||
optimizationPlan
|
||||
- sections to edit
|
||||
- sections to leave unchanged
|
||||
- keywords to add
|
||||
- keywords to reduce
|
||||
- media SEO tasks
|
||||
- length risks
|
||||
- tone risks
|
||||
```
|
||||
|
||||
UI показывает план и просит подтверждение.
|
||||
|
||||
Без подтверждения plan не переходит в rewrite.
|
||||
|
||||
## 11. Rewrite Workspace
|
||||
|
||||
Пользователь выбирает секцию.
|
||||
|
||||
UI показывает:
|
||||
|
||||
```text
|
||||
left:
|
||||
- originalText
|
||||
- current workingText
|
||||
|
||||
right:
|
||||
- assigned keywords
|
||||
- section meaning
|
||||
- limits
|
||||
- warnings
|
||||
- Codex variants
|
||||
```
|
||||
|
||||
Codex предлагает варианты, но не меняет target project.
|
||||
|
||||
Пользователь может:
|
||||
|
||||
- принять вариант;
|
||||
- смешать варианты;
|
||||
- отредактировать руками;
|
||||
- попросить другой тон;
|
||||
- попросить ужать текст;
|
||||
- отменить изменения в working draft.
|
||||
|
||||
## 12. Media SEO
|
||||
|
||||
Media SEO может идти как отдельный экран или как часть плана оптимизации.
|
||||
|
||||
Система находит:
|
||||
|
||||
```text
|
||||
img
|
||||
picture/source/srcset
|
||||
video/source/poster
|
||||
CSS background images
|
||||
filename
|
||||
alt/title/aria
|
||||
captions/context
|
||||
```
|
||||
|
||||
Для каждого media item:
|
||||
|
||||
```text
|
||||
status:
|
||||
ok
|
||||
missing
|
||||
weak
|
||||
duplicate
|
||||
decorative
|
||||
needs-human
|
||||
```
|
||||
|
||||
Codex может предложить:
|
||||
|
||||
- alt;
|
||||
- caption;
|
||||
- video description;
|
||||
- filename suggestion.
|
||||
|
||||
Правило:
|
||||
|
||||
```text
|
||||
ключ можно встроить в alt/filename/caption,
|
||||
если он естественно описывает изображение и контекст секции.
|
||||
```
|
||||
|
||||
Не набивать ключи без смысла.
|
||||
|
||||
## 13. Final Validation
|
||||
|
||||
После текстовых и media-правок запускаются:
|
||||
|
||||
```text
|
||||
checkOverseo:
|
||||
- перетошнота
|
||||
- заспамленность
|
||||
- водность
|
||||
- повторы
|
||||
- proximity
|
||||
- keyword stuffing в alt/filename
|
||||
|
||||
ru-text:
|
||||
- тон
|
||||
- читаемость
|
||||
- канцелярит
|
||||
- typographic/editorial issues
|
||||
|
||||
structure checks:
|
||||
- title/description/h1/h2
|
||||
- section meaning conflict
|
||||
- design length limit
|
||||
|
||||
media checks:
|
||||
- missing/weak/duplicate alt
|
||||
- decorative media
|
||||
- video context
|
||||
- filename rename plan
|
||||
```
|
||||
|
||||
UI показывает:
|
||||
|
||||
- общий статус;
|
||||
- ошибки;
|
||||
- предупреждения;
|
||||
- diff `originalText` vs `workingText`;
|
||||
- media diff;
|
||||
- что нужно поправить до apply.
|
||||
|
||||
Важно: любые media-правки тоже проходят финальную проверку. Нельзя сделать Media SEO и сразу Apply без Final Validation.
|
||||
|
||||
## 14. Apply Changes
|
||||
|
||||
Apply — ручной шаг.
|
||||
|
||||
Перед применением:
|
||||
|
||||
```text
|
||||
UI shows:
|
||||
- files affected
|
||||
- sections affected
|
||||
- text diff
|
||||
- media diff
|
||||
- filename rename plan
|
||||
- validation status
|
||||
```
|
||||
|
||||
Пользователь нажимает `Apply to target project`.
|
||||
|
||||
Приложение:
|
||||
|
||||
- создаёт changeset;
|
||||
- делает dry-run;
|
||||
- сохраняет backup affected files;
|
||||
- применяет изменения в target project;
|
||||
- проверяет rename/update references;
|
||||
- делает rollback, если критический шаг упал;
|
||||
- сохраняет отчёт;
|
||||
- сохраняет run history.
|
||||
|
||||
Codex должен получать жёсткое ограничение:
|
||||
|
||||
```text
|
||||
Do not modify application code.
|
||||
Only modify approved text/media fields in target project:
|
||||
- visible text
|
||||
- title/description
|
||||
- headings
|
||||
- alt/title/aria
|
||||
- captions
|
||||
- approved filenames
|
||||
```
|
||||
|
||||
## 15. Export
|
||||
|
||||
После apply пользователь может сделать export:
|
||||
|
||||
```text
|
||||
Export options:
|
||||
- zip target project
|
||||
- export reports
|
||||
- export keyword map
|
||||
- export changeset summary
|
||||
```
|
||||
|
||||
Это нужно для ручной заливки на хостинг.
|
||||
|
||||
Автопубликации нет.
|
||||
|
||||
## 16. Project History
|
||||
|
||||
В проекте должна сохраняться история:
|
||||
|
||||
```text
|
||||
history:
|
||||
- scans
|
||||
- baseline audits
|
||||
- Wordstat collections
|
||||
- keyword cleaning decisions
|
||||
- keyword maps
|
||||
- optimization plans
|
||||
- rewrites
|
||||
- validations
|
||||
- changesets
|
||||
- exports
|
||||
```
|
||||
|
||||
Это нужно, чтобы:
|
||||
|
||||
- вернуться к работе позже;
|
||||
- понять, что уже сделано;
|
||||
- подключить Яндекс Метрику/Вебмастер в будущем;
|
||||
- делать дальнейшие SEO-корректировки.
|
||||
|
||||
## 17. Local Storage
|
||||
|
||||
MVP локальный, но проектный.
|
||||
|
||||
Рекомендуемая модель:
|
||||
|
||||
```text
|
||||
Postgres:
|
||||
- projects
|
||||
- project_sources
|
||||
- scan_versions
|
||||
- pages / page_versions
|
||||
- sections / section_versions
|
||||
- media_assets / media_versions
|
||||
- runs
|
||||
- seeds
|
||||
- wordstat_jobs / wordstat_results
|
||||
- keyword_decisions
|
||||
- keyword_map_items
|
||||
- drafts / draft_items
|
||||
- validations / validation_issues
|
||||
- changesets / changeset_items
|
||||
- exports
|
||||
|
||||
Object storage:
|
||||
- raw dumps
|
||||
- reports
|
||||
- snapshots
|
||||
- previews / video frames
|
||||
- backups
|
||||
- export archives
|
||||
```
|
||||
|
||||
Правило:
|
||||
|
||||
```text
|
||||
Postgres = source of truth
|
||||
Object storage = artifacts
|
||||
YAML/JSON/MD = export/debug only
|
||||
```
|
||||
|
||||
## 18. Main UI Navigation
|
||||
|
||||
```text
|
||||
Projects
|
||||
-> Project Dashboard
|
||||
-> Scan / Baseline Audit
|
||||
-> Pages
|
||||
-> Seeds
|
||||
-> Wordstat
|
||||
-> Keyword Cleaning
|
||||
-> Keyword Map
|
||||
-> Optimization Plan
|
||||
-> Rewrite
|
||||
-> Media SEO
|
||||
-> Final Validation
|
||||
-> Apply / Export
|
||||
-> History / Reports
|
||||
```
|
||||
|
||||
## 19. What Is Not MVP
|
||||
|
||||
- создание текстов с нуля;
|
||||
- автопостинг;
|
||||
- Яндекс Метрика / Вебмастер;
|
||||
- полноценная версионность draft history;
|
||||
- Codex skills;
|
||||
- облачная версия;
|
||||
- командная работа.
|
||||
|
||||
Эти слои можно добавить позже, когда базовый проектный SEO-цикл стабилен.
|
||||
|
||||
## 20. Related Development Guardrails
|
||||
|
||||
Технический порядок backend pipeline, зоны ответственности инструментов и риски разработки вынесены в `IMPLEMENTATION_GUARDRAILS.md`.
|
||||
|
||||
Важно: этот user flow описывает, как процесс видит пользователь. Внутренний порядок выполнения должен соответствовать guardrails.
|
||||
|
||||
UI/UX-набор обязательных экранов, компонентов и состояний вынесен в `UI_UX_SPEC.md`.
|
||||
|
||||
Пошаговый план реализации с критериями готовности вынесен в `TASK_MANAGER.md`.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>seo_mode</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "@seo-mode/app",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite --host 127.0.0.1",
|
||||
"build": "tsc -p tsconfig.json && vite build",
|
||||
"preview": "vite preview --host 127.0.0.1",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@xyflow/react": "^12.11.1",
|
||||
"lucide-react": "^0.561.0",
|
||||
"react": "^19.2.3",
|
||||
"react-dom": "^19.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.2.7",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^5.1.1",
|
||||
"typescript": "^5.9.3",
|
||||
"vite": "^8.0.16"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,9 @@
|
|||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { App } from "./App";
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>
|
||||
);
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1 @@
|
|||
/// <reference types="vite/client" />
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"extends": "../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"jsx": "react-jsx",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "src/**/*.tsx", "vite.config.ts"]
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
import react from "@vitejs/plugin-react";
|
||||
import { defineConfig } from "vite";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 5173,
|
||||
strictPort: false
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# Локальная инфраструктура
|
||||
|
||||
Эта папка содержит локальную инфраструктуру разработки для seo_mode.
|
||||
|
||||
```powershell
|
||||
docker compose -f infra/docker-compose.yml up -d
|
||||
```
|
||||
|
||||
Сервисы:
|
||||
|
||||
- Postgres: `localhost:5432`
|
||||
- MinIO API: `localhost:9000`
|
||||
- MinIO console: `localhost:9001`
|
||||
|
||||
Используй `.env.example` как базовый пример локальных переменных окружения.
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: seo-farm-postgres
|
||||
environment:
|
||||
POSTGRES_DB: seo_farm
|
||||
POSTGRES_USER: seo_farm
|
||||
POSTGRES_PASSWORD: seo_farm
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U seo_farm -d seo_farm"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: seo-farm-minio
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER: seo_farm
|
||||
MINIO_ROOT_PASSWORD: seo_farm_secret
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
minio-data:
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "seo-mode",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
"app",
|
||||
"server"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "concurrently -n app,server -c cyan,green \"npm run dev -w app\" \"npm run dev -w server\"",
|
||||
"dev:app": "npm run dev -w app",
|
||||
"dev:server": "npm run dev -w server",
|
||||
"build": "npm run build -w server && npm run build -w app",
|
||||
"typecheck": "npm run typecheck -w server && npm run typecheck -w app",
|
||||
"db:migrate": "npm run db:migrate -w server"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.2.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
create extension if not exists pgcrypto;
|
||||
|
||||
create type source_type as enum ('local_folder', 'git_repo', 'zip_upload', 'site_crawl');
|
||||
create type run_status as enum ('queued', 'running', 'done', 'failed', 'cancelled');
|
||||
create type reconcile_status as enum ('matched', 'changed', 'new', 'orphaned', 'conflicted');
|
||||
|
||||
create table projects (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
name text not null,
|
||||
slug text not null unique,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table project_sources (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
type source_type not null,
|
||||
config jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table scan_versions (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
source_id uuid not null references project_sources(id) on delete cascade,
|
||||
status run_status not null default 'queued',
|
||||
project_index_key text,
|
||||
started_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
error_message text,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table pages (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
logical_key text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
unique(project_id, logical_key)
|
||||
);
|
||||
|
||||
create table page_versions (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
page_id uuid not null references pages(id) on delete cascade,
|
||||
scan_version_id uuid not null references scan_versions(id) on delete cascade,
|
||||
source_path text not null,
|
||||
url_path text,
|
||||
title text,
|
||||
description text,
|
||||
fingerprint text,
|
||||
reconcile_status reconcile_status not null default 'new',
|
||||
raw jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table sections (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
page_id uuid references pages(id) on delete cascade,
|
||||
logical_key text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
unique(project_id, logical_key)
|
||||
);
|
||||
|
||||
create table section_versions (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
section_id uuid not null references sections(id) on delete cascade,
|
||||
scan_version_id uuid not null references scan_versions(id) on delete cascade,
|
||||
selector text,
|
||||
heading text,
|
||||
normalized_text_hash text,
|
||||
selector_fingerprint text,
|
||||
reconcile_status reconcile_status not null default 'new',
|
||||
raw jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table media_assets (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
logical_key text not null,
|
||||
created_at timestamptz not null default now(),
|
||||
unique(project_id, logical_key)
|
||||
);
|
||||
|
||||
create table media_versions (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
media_id uuid not null references media_assets(id) on delete cascade,
|
||||
scan_version_id uuid not null references scan_versions(id) on delete cascade,
|
||||
source_path text not null,
|
||||
media_type text not null,
|
||||
section_id uuid references sections(id) on delete set null,
|
||||
reconcile_status reconcile_status not null default 'new',
|
||||
raw jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table runs (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
run_type text not null,
|
||||
status run_status not null default 'queued',
|
||||
input jsonb not null default '{}'::jsonb,
|
||||
output jsonb,
|
||||
error_message text,
|
||||
started_at timestamptz,
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table seeds (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
section_id uuid references sections(id) on delete set null,
|
||||
phrase text not null,
|
||||
source text not null default 'model',
|
||||
selected boolean not null default true,
|
||||
reason text,
|
||||
confidence numeric(5, 4),
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table wordstat_jobs (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
seed_id uuid references seeds(id) on delete set null,
|
||||
status run_status not null default 'queued',
|
||||
raw_result_key text,
|
||||
created_at timestamptz not null default now(),
|
||||
completed_at timestamptz
|
||||
);
|
||||
|
||||
create table wordstat_results (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
job_id uuid not null references wordstat_jobs(id) on delete cascade,
|
||||
phrase text not null,
|
||||
frequency integer,
|
||||
frequency_group text,
|
||||
raw jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table keyword_decisions (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
wordstat_result_id uuid references wordstat_results(id) on delete set null,
|
||||
phrase text not null,
|
||||
status text not null,
|
||||
reason text,
|
||||
approved boolean not null default false,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table keyword_map_items (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
section_id uuid references sections(id) on delete cascade,
|
||||
keyword_decision_id uuid references keyword_decisions(id) on delete cascade,
|
||||
role text not null,
|
||||
exact_limit integer,
|
||||
soft_forms jsonb not null default '[]'::jsonb,
|
||||
approved boolean not null default false,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table optimization_plans (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
payload jsonb not null default '{}'::jsonb,
|
||||
approved boolean not null default false,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table drafts (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
page_id uuid references pages(id) on delete cascade,
|
||||
status text not null default 'active',
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table draft_items (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
draft_id uuid not null references drafts(id) on delete cascade,
|
||||
section_id uuid references sections(id) on delete cascade,
|
||||
field text not null,
|
||||
original_snapshot_key text,
|
||||
working_snapshot_key text,
|
||||
working_text text,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table validations (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
draft_id uuid references drafts(id) on delete set null,
|
||||
status text not null,
|
||||
summary jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table validation_issues (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
validation_id uuid not null references validations(id) on delete cascade,
|
||||
severity text not null,
|
||||
code text not null,
|
||||
message text not null,
|
||||
source_ref jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table changesets (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
draft_id uuid references drafts(id) on delete set null,
|
||||
validation_id uuid references validations(id) on delete set null,
|
||||
status text not null default 'pending',
|
||||
backup_key text,
|
||||
summary jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
applied_at timestamptz
|
||||
);
|
||||
|
||||
create table changeset_items (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
changeset_id uuid not null references changesets(id) on delete cascade,
|
||||
change_type text not null,
|
||||
source_path text not null,
|
||||
selector_or_media_id text,
|
||||
before_value text,
|
||||
after_value text,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
||||
create table exports (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
project_id uuid not null references projects(id) on delete cascade,
|
||||
export_type text not null,
|
||||
object_key text not null,
|
||||
created_at timestamptz not null default now()
|
||||
);
|
||||
|
|
@ -0,0 +1 @@
|
|||
alter type source_type add value if not exists 'browser_folder';
|
||||
|
|
@ -0,0 +1 @@
|
|||
create unique index if not exists projects_name_unique_ci on projects (lower(name));
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
alter table pages
|
||||
add column if not exists selected boolean not null default false;
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
alter table seeds
|
||||
add column if not exists semantic_run_id uuid references runs(id) on delete set null,
|
||||
add column if not exists cluster_id text,
|
||||
add column if not exists cluster_title text,
|
||||
add column if not exists priority text,
|
||||
add column if not exists updated_at timestamptz not null default now();
|
||||
|
||||
create unique index if not exists seeds_project_semantic_phrase_cluster_idx
|
||||
on seeds(project_id, semantic_run_id, lower(phrase), coalesce(cluster_id, ''));
|
||||
|
||||
alter table wordstat_jobs
|
||||
add column if not exists semantic_run_id uuid references runs(id) on delete set null,
|
||||
add column if not exists provider text not null default 'disabled',
|
||||
add column if not exists mode text not null default 'disabled',
|
||||
add column if not exists region text not null default 'ru',
|
||||
add column if not exists requested_phrases jsonb not null default '[]'::jsonb,
|
||||
add column if not exists result_count integer not null default 0,
|
||||
add column if not exists error_message text,
|
||||
add column if not exists started_at timestamptz,
|
||||
add column if not exists updated_at timestamptz not null default now();
|
||||
|
||||
create index if not exists wordstat_jobs_project_semantic_created_idx
|
||||
on wordstat_jobs(project_id, semantic_run_id, created_at desc);
|
||||
|
||||
alter table wordstat_results
|
||||
add column if not exists seed_id uuid references seeds(id) on delete set null,
|
||||
add column if not exists source_phrase text,
|
||||
add column if not exists normalized_phrase text,
|
||||
add column if not exists region text,
|
||||
add column if not exists position integer;
|
||||
|
||||
create index if not exists wordstat_results_job_seed_idx
|
||||
on wordstat_results(job_id, seed_id);
|
||||
|
||||
create index if not exists wordstat_results_phrase_idx
|
||||
on wordstat_results(lower(phrase));
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
create table if not exists external_service_settings (
|
||||
service_id text primary key,
|
||||
enabled boolean not null default false,
|
||||
mode text not null default 'disabled',
|
||||
public_config jsonb not null default '{}'::jsonb,
|
||||
secret_config jsonb not null default '{}'::jsonb,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
{
|
||||
"name": "@seo-mode/server",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "node dist/index.js",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.json",
|
||||
"db:migrate": "tsx src/scripts/migrate.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.946.0",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^17.2.3",
|
||||
"express": "^5.2.1",
|
||||
"pg": "^8.16.3",
|
||||
"playwright-core": "^1.61.1",
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/node": "^24.10.2",
|
||||
"@types/pg": "^8.16.0",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,73 @@
|
|||
import "dotenv/config";
|
||||
import { z } from "zod";
|
||||
|
||||
const booleanFromString = z.preprocess((value) => {
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value !== "string") return value;
|
||||
return ["1", "true", "yes"].includes(value.toLowerCase());
|
||||
}, z.boolean());
|
||||
|
||||
const envSchema = z.object({
|
||||
SERVER_PORT: z.coerce.number().int().positive().default(4100),
|
||||
DATABASE_URL: z.string().url().default("postgres://seo_farm:seo_farm@localhost:5432/seo_farm"),
|
||||
S3_ENDPOINT: z.string().url().default("http://localhost:9000"),
|
||||
S3_REGION: z.string().default("us-east-1"),
|
||||
S3_ACCESS_KEY_ID: z.string().default("seo_farm"),
|
||||
S3_SECRET_ACCESS_KEY: z.string().default("seo_farm_secret"),
|
||||
S3_BUCKET: z.string().default("seo-farm-artifacts"),
|
||||
S3_FORCE_PATH_STYLE: booleanFromString.default(true),
|
||||
WORDSTAT_PROVIDER: z.enum(["disabled", "mcp_kv", "yandex_api"]).default("disabled"),
|
||||
WORDSTAT_MCP_ENDPOINT: z.string().url().optional(),
|
||||
WORDSTAT_API_BASE_URL: z.string().url().optional(),
|
||||
WORDSTAT_API_TOKEN: z.string().optional(),
|
||||
WORDSTAT_AUTH_TYPE: z.enum(["api_key", "iam_token"]).default("api_key"),
|
||||
WORDSTAT_FOLDER_ID: z.string().optional(),
|
||||
WORDSTAT_REGION_IDS: z.string().default("225"),
|
||||
WORDSTAT_DEVICES: z.string().default("DEVICE_ALL"),
|
||||
WORDSTAT_NUM_PHRASES: z.coerce.number().int().positive().default(50),
|
||||
SERP_PROVIDER: z.enum(["disabled", "external_api"]).default("disabled"),
|
||||
SERP_API_BASE_URL: z.string().url().optional(),
|
||||
SERP_API_TOKEN: z.string().optional(),
|
||||
PREVIEW_CHROME_PATH: z.string().optional(),
|
||||
PREVIEW_SNAPSHOT_CONCURRENCY: z.coerce.number().int().positive().max(4).default(2),
|
||||
PREVIEW_SNAPSHOT_WAIT_MS: z.coerce.number().int().nonnegative().max(10_000).default(1400)
|
||||
});
|
||||
|
||||
const parsedEnv = envSchema.parse(process.env);
|
||||
|
||||
export const env = {
|
||||
serverPort: parsedEnv.SERVER_PORT,
|
||||
databaseUrl: parsedEnv.DATABASE_URL,
|
||||
s3: {
|
||||
endpoint: parsedEnv.S3_ENDPOINT,
|
||||
region: parsedEnv.S3_REGION,
|
||||
accessKeyId: parsedEnv.S3_ACCESS_KEY_ID,
|
||||
secretAccessKey: parsedEnv.S3_SECRET_ACCESS_KEY,
|
||||
bucket: parsedEnv.S3_BUCKET,
|
||||
forcePathStyle: parsedEnv.S3_FORCE_PATH_STYLE
|
||||
},
|
||||
marketProviders: {
|
||||
wordstat: {
|
||||
provider: parsedEnv.WORDSTAT_PROVIDER,
|
||||
mcpEndpoint: parsedEnv.WORDSTAT_MCP_ENDPOINT,
|
||||
apiBaseUrl: parsedEnv.WORDSTAT_API_BASE_URL,
|
||||
apiToken: parsedEnv.WORDSTAT_API_TOKEN,
|
||||
authType: parsedEnv.WORDSTAT_AUTH_TYPE,
|
||||
folderId: parsedEnv.WORDSTAT_FOLDER_ID,
|
||||
regionIds: parsedEnv.WORDSTAT_REGION_IDS.split(",").map((item) => item.trim()).filter(Boolean),
|
||||
devices: parsedEnv.WORDSTAT_DEVICES.split(",").map((item) => item.trim()).filter(Boolean),
|
||||
numPhrases: parsedEnv.WORDSTAT_NUM_PHRASES,
|
||||
hasApiToken: Boolean(parsedEnv.WORDSTAT_API_TOKEN)
|
||||
},
|
||||
serp: {
|
||||
provider: parsedEnv.SERP_PROVIDER,
|
||||
apiBaseUrl: parsedEnv.SERP_API_BASE_URL,
|
||||
hasApiToken: Boolean(parsedEnv.SERP_API_TOKEN)
|
||||
}
|
||||
},
|
||||
preview: {
|
||||
chromePath: parsedEnv.PREVIEW_CHROME_PATH || "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
snapshotConcurrency: parsedEnv.PREVIEW_SNAPSHOT_CONCURRENCY,
|
||||
snapshotWaitMs: parsedEnv.PREVIEW_SNAPSHOT_WAIT_MS
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import pg from "pg";
|
||||
import { env } from "../config/env.js";
|
||||
|
||||
export const pool = new pg.Pool({
|
||||
connectionString: env.databaseUrl
|
||||
});
|
||||
|
||||
export async function checkDatabase() {
|
||||
const result = await pool.query<{ now: Date }>("select now() as now");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
export async function closeDatabase() {
|
||||
await pool.end();
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import cors from "cors";
|
||||
import express from "express";
|
||||
import { checkDatabase } from "../db/client.js";
|
||||
import { previewRouter } from "../preview/routes.js";
|
||||
import { projectsRouter } from "../projects/routes.js";
|
||||
import { settingsRouter } from "../settings/routes.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
|
||||
export function createServer() {
|
||||
const app = express();
|
||||
|
||||
app.use(cors({ exposedHeaders: ["Content-Disposition"] }));
|
||||
app.use(express.json({ limit: "150mb" }));
|
||||
|
||||
app.use("/projects", projectsRouter);
|
||||
app.use("/preview", previewRouter);
|
||||
app.use("/settings", settingsRouter);
|
||||
|
||||
app.get("/health", (_request, response) => {
|
||||
response.json({
|
||||
status: "ok",
|
||||
service: "seo_mode-server"
|
||||
});
|
||||
});
|
||||
|
||||
app.get("/health/deep", async (_request, response) => {
|
||||
const checks: Record<string, unknown> = {};
|
||||
|
||||
try {
|
||||
checks.database = await checkDatabase();
|
||||
} catch (error) {
|
||||
checks.database = { error: error instanceof Error ? error.message : "unknown error" };
|
||||
}
|
||||
|
||||
try {
|
||||
await storageProvider.ensureReady();
|
||||
checks.objectStorage = { status: "ok" };
|
||||
} catch (error) {
|
||||
checks.objectStorage = { error: error instanceof Error ? error.message : "unknown error" };
|
||||
}
|
||||
|
||||
const hasError = Object.values(checks).some((value) => {
|
||||
return typeof value === "object" && value !== null && "error" in value;
|
||||
});
|
||||
|
||||
response.status(hasError ? 503 : 200).json({
|
||||
status: hasError ? "degraded" : "ok",
|
||||
checks
|
||||
});
|
||||
});
|
||||
|
||||
return app;
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import { env } from "./config/env.js";
|
||||
import { createServer } from "./http/server.js";
|
||||
|
||||
const app = createServer();
|
||||
|
||||
app.listen(env.serverPort, () => {
|
||||
console.log(`seo_mode server listening on http://localhost:${env.serverPort}`);
|
||||
});
|
||||
|
|
@ -0,0 +1,372 @@
|
|||
import { env } from "../config/env.js";
|
||||
import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js";
|
||||
import { collectWordstatEvidence, getLatestWordstatEvidence, type WordstatEvidence } from "./wordstatRepository.js";
|
||||
import { createWordstatProvider } from "./wordstatProvider.js";
|
||||
|
||||
type MarketProviderId = "model" | "serp" | "text_quality" | "wordstat";
|
||||
type MarketProviderStatus = "connected" | "disabled" | "not_configured" | "not_implemented";
|
||||
type MarketEnrichmentState = "not_ready" | "not_collected" | "not_configured" | "partial_ready";
|
||||
type MarketEvidenceStatus =
|
||||
| "collection_failed"
|
||||
| "collected"
|
||||
| "collected_no_signal"
|
||||
| "collected_related_only"
|
||||
| "deterministic_only"
|
||||
| "not_collected"
|
||||
| "not_configured"
|
||||
| "ready_for_collection";
|
||||
|
||||
type MarketProviderDescriptor = {
|
||||
id: MarketProviderId;
|
||||
label: string;
|
||||
role: string;
|
||||
status: MarketProviderStatus;
|
||||
mode: string;
|
||||
userActionRequired: boolean;
|
||||
platformActionRequired: boolean;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type MarketEnrichmentContract = {
|
||||
schemaVersion: "market-enrichment.v1";
|
||||
projectId: string;
|
||||
semanticRunId: string | null;
|
||||
generatedAt: string;
|
||||
state: MarketEnrichmentState;
|
||||
readiness: {
|
||||
canCollectWordstat: boolean;
|
||||
canCollectSerp: boolean;
|
||||
seedCount: number;
|
||||
briefCount: number;
|
||||
configuredProviderCount: number;
|
||||
userSetupRequired: boolean;
|
||||
};
|
||||
providers: MarketProviderDescriptor[];
|
||||
seedQueue: Array<{
|
||||
phrase: string;
|
||||
clusterId: string;
|
||||
clusterTitle: string;
|
||||
priority: "high" | "medium" | "low";
|
||||
source: "detected" | "ontology";
|
||||
evidenceStatus: MarketEvidenceStatus;
|
||||
targetProviders: MarketProviderId[];
|
||||
blocker: string | null;
|
||||
frequency: number | null;
|
||||
frequencyGroup: string | null;
|
||||
relatedCount: number;
|
||||
lastCollectedAt: string | null;
|
||||
}>;
|
||||
briefEvidence: Array<{
|
||||
path: string;
|
||||
priority: "high" | "medium" | "low";
|
||||
action: "create" | "strengthen" | "validate";
|
||||
qualityScore: number | null;
|
||||
qualityLevel: "blocked" | "needs_work" | "ready" | null;
|
||||
seedPhrases: string[];
|
||||
evidenceStatus: MarketEvidenceStatus;
|
||||
requiredProviders: MarketProviderId[];
|
||||
missingEvidence: string[];
|
||||
}>;
|
||||
wordstat: WordstatEvidence;
|
||||
nextActions: string[];
|
||||
};
|
||||
|
||||
function isSerpConfigured() {
|
||||
const config = env.marketProviders.serp;
|
||||
|
||||
return config.provider !== "disabled" && Boolean(config.apiBaseUrl && config.hasApiToken);
|
||||
}
|
||||
|
||||
async function buildProviderRegistry(): Promise<MarketProviderDescriptor[]> {
|
||||
const serpConfig = env.marketProviders.serp;
|
||||
const wordstatStatus = (await createWordstatProvider()).getStatus();
|
||||
const serpConfigured = isSerpConfigured();
|
||||
|
||||
return [
|
||||
{
|
||||
id: "wordstat",
|
||||
label: "Yandex Wordstat",
|
||||
role: "Частотность, похожие запросы, региональные и сезонные сигналы по seed-фразам.",
|
||||
status: wordstatStatus.connected ? "connected" : "not_configured",
|
||||
mode: wordstatStatus.mode,
|
||||
userActionRequired: false,
|
||||
platformActionRequired: !wordstatStatus.connected,
|
||||
message: wordstatStatus.connected
|
||||
? wordstatStatus.message
|
||||
: `${wordstatStatus.message} Пользовательский сценарий не должен требовать ручной настройки MCP.`
|
||||
},
|
||||
{
|
||||
id: "serp",
|
||||
label: "Yandex SERP",
|
||||
role: "Типы страниц в выдаче, конкурентные углы, сложность и сниппеты.",
|
||||
status: serpConfigured ? "connected" : "not_configured",
|
||||
mode: serpConfig.provider,
|
||||
userActionRequired: false,
|
||||
platformActionRequired: !serpConfigured,
|
||||
message: serpConfigured
|
||||
? "SERP provider настроен; можно собирать конкурентные evidence."
|
||||
: "SERP provider ещё не настроен; deterministic analysis не должен притворяться рыночной проверкой."
|
||||
},
|
||||
{
|
||||
id: "model",
|
||||
label: "Model Provider",
|
||||
role: "Классификация неоднозначных фраз, интентов и конкурентных формулировок.",
|
||||
status: "not_implemented",
|
||||
mode: "provider_abstraction",
|
||||
userActionRequired: false,
|
||||
platformActionRequired: true,
|
||||
message: "Model provider будет подключаться платформенно через общий AI layer; в MVP решение не просит пользователя о ключах."
|
||||
},
|
||||
{
|
||||
id: "text_quality",
|
||||
label: "Text Quality",
|
||||
role: "Локальная проверка заспамленности, воды и перебора ключей после рерайта.",
|
||||
status: "not_implemented",
|
||||
mode: "local_ru_text",
|
||||
userActionRequired: false,
|
||||
platformActionRequired: true,
|
||||
message: "Text.ru API не подключается; по ТЗ нужен локальный quality checker по аналогичной модели показателей."
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
function getProviderStatus(providers: MarketProviderDescriptor[], id: MarketProviderId) {
|
||||
return providers.find((provider) => provider.id === id)?.status ?? "not_configured";
|
||||
}
|
||||
|
||||
function getEvidenceStatus(providers: MarketProviderDescriptor[], requiredProviders: MarketProviderId[]): MarketEvidenceStatus {
|
||||
if (requiredProviders.length === 0) {
|
||||
return "deterministic_only";
|
||||
}
|
||||
|
||||
const statuses = requiredProviders.map((providerId) => getProviderStatus(providers, providerId));
|
||||
|
||||
if (statuses.some((status) => status === "not_configured" || status === "disabled")) {
|
||||
return "not_configured";
|
||||
}
|
||||
|
||||
if (statuses.every((status) => status === "connected")) {
|
||||
return "ready_for_collection";
|
||||
}
|
||||
|
||||
return "not_collected";
|
||||
}
|
||||
|
||||
function getSeedBlocker(evidenceStatus: MarketEvidenceStatus) {
|
||||
if (evidenceStatus === "not_configured") {
|
||||
return "Нужен платформенный Wordstat provider; пользователь не должен вручную подключать MCP или внешний аккаунт.";
|
||||
}
|
||||
|
||||
if (evidenceStatus === "collection_failed") {
|
||||
return "Последний Wordstat job упал; смотри provider error и raw job status в dev-mode.";
|
||||
}
|
||||
|
||||
if (
|
||||
evidenceStatus === "ready_for_collection" ||
|
||||
evidenceStatus === "collected" ||
|
||||
evidenceStatus === "collected_related_only" ||
|
||||
evidenceStatus === "collected_no_signal"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return "Сбор ещё не запускался.";
|
||||
}
|
||||
|
||||
function normalizePhrase(value: string) {
|
||||
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
}
|
||||
|
||||
function buildWordstatSeedMap(wordstat: WordstatEvidence) {
|
||||
return new Map(wordstat.seedResults.map((result) => [normalizePhrase(result.phrase), result]));
|
||||
}
|
||||
|
||||
function getSeedEvidenceStatus(
|
||||
providers: MarketProviderDescriptor[],
|
||||
wordstat: WordstatEvidence,
|
||||
wordstatResult: WordstatEvidence["seedResults"][number] | undefined
|
||||
): MarketEvidenceStatus {
|
||||
if (wordstatResult?.frequency !== null && wordstatResult?.frequency !== undefined) {
|
||||
return "collected";
|
||||
}
|
||||
|
||||
if (wordstat.latestJob?.status === "failed") {
|
||||
return "collection_failed";
|
||||
}
|
||||
|
||||
if (wordstat.latestJob?.status === "done" && wordstatResult && wordstatResult.relatedCount > 0) {
|
||||
return "collected_related_only";
|
||||
}
|
||||
|
||||
if (wordstat.latestJob?.status === "done") {
|
||||
return "collected_no_signal";
|
||||
}
|
||||
|
||||
return getEvidenceStatus(providers, ["wordstat"]);
|
||||
}
|
||||
|
||||
function buildSeedQueue(analysis: SemanticAnalysisRun, providers: MarketProviderDescriptor[], wordstat: WordstatEvidence) {
|
||||
const wordstatResults = buildWordstatSeedMap(wordstat);
|
||||
|
||||
return analysis.seedCandidates.slice(0, 80).map((seed) => {
|
||||
const targetProviders: MarketProviderId[] = ["wordstat"];
|
||||
const result = wordstatResults.get(normalizePhrase(seed.phrase));
|
||||
const evidenceStatus = getSeedEvidenceStatus(providers, wordstat, result);
|
||||
|
||||
return {
|
||||
phrase: seed.phrase,
|
||||
clusterId: seed.clusterId,
|
||||
clusterTitle: seed.clusterTitle,
|
||||
priority: seed.priority,
|
||||
source: seed.source,
|
||||
evidenceStatus,
|
||||
targetProviders,
|
||||
blocker: getSeedBlocker(evidenceStatus),
|
||||
frequency: result?.frequency ?? null,
|
||||
frequencyGroup: result?.frequencyGroup ?? null,
|
||||
relatedCount: result?.relatedCount ?? 0,
|
||||
lastCollectedAt: result?.collectedAt ?? null
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function buildBriefEvidence(analysis: SemanticAnalysisRun, providers: MarketProviderDescriptor[]) {
|
||||
return (analysis.landingPageBriefs ?? []).slice(0, 80).map((brief) => {
|
||||
const requiredProviders: MarketProviderId[] = ["wordstat", "serp"];
|
||||
const evidenceStatus = getEvidenceStatus(providers, requiredProviders);
|
||||
|
||||
return {
|
||||
path: brief.path,
|
||||
priority: brief.priority,
|
||||
action: brief.action,
|
||||
qualityScore: brief.qualityContract?.score ?? null,
|
||||
qualityLevel: brief.qualityContract?.level ?? null,
|
||||
seedPhrases: brief.seedPhrases.slice(0, 8),
|
||||
evidenceStatus,
|
||||
requiredProviders,
|
||||
missingEvidence: brief.qualityContract?.missingEvidence ?? [
|
||||
"Wordstat частотность и сезонность",
|
||||
"SERP-конкуренты и типы страниц",
|
||||
"Региональная выдача"
|
||||
]
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function getEnrichmentState(
|
||||
analysis: SemanticAnalysisRun | null,
|
||||
providers: MarketProviderDescriptor[],
|
||||
seedCount: number,
|
||||
wordstat: WordstatEvidence
|
||||
): MarketEnrichmentState {
|
||||
if (!analysis) {
|
||||
return "not_ready";
|
||||
}
|
||||
|
||||
if (seedCount === 0) {
|
||||
return "not_ready";
|
||||
}
|
||||
|
||||
if (getProviderStatus(providers, "wordstat") !== "connected") {
|
||||
return "not_configured";
|
||||
}
|
||||
|
||||
if (wordstat.latestJob?.status === "done" && wordstat.summary.collectedSeedCount > 0) {
|
||||
return getProviderStatus(providers, "serp") === "connected" ? "not_collected" : "partial_ready";
|
||||
}
|
||||
|
||||
if (getProviderStatus(providers, "serp") !== "connected") {
|
||||
return "partial_ready";
|
||||
}
|
||||
|
||||
return "not_collected";
|
||||
}
|
||||
|
||||
function buildNextActions(contract: Omit<MarketEnrichmentContract, "nextActions">) {
|
||||
const actions: string[] = [];
|
||||
|
||||
if (!contract.semanticRunId) {
|
||||
actions.push("Сначала запустить semantic analysis: без seed candidates market layer не стартует.");
|
||||
return actions;
|
||||
}
|
||||
|
||||
const wordstat = contract.providers.find((provider) => provider.id === "wordstat");
|
||||
const serp = contract.providers.find((provider) => provider.id === "serp");
|
||||
|
||||
if (wordstat?.status !== "connected") {
|
||||
actions.push("Настроить платформенный Wordstat provider: MCP-KV endpoint или прямой API adapter на backend.");
|
||||
} else if (contract.wordstat.latestJob?.status === "done") {
|
||||
actions.push(
|
||||
`Wordstat evidence собрано: ${contract.wordstat.summary.collectedSeedCount} seed-фраз, ${contract.wordstat.summary.resultCount} rows. Следующий слой — keywordService cleaning.`
|
||||
);
|
||||
} else if (contract.wordstat.latestJob?.status === "failed") {
|
||||
actions.push("Проверить Wordstat provider: последний job завершился ошибкой, raw/error сохранены в dev-mode contract.");
|
||||
} else if (contract.readiness.seedCount > 0) {
|
||||
actions.push(`Запустить Wordstat collection для ${contract.readiness.seedCount} seed-фраз.`);
|
||||
}
|
||||
|
||||
if (serp?.status !== "connected") {
|
||||
actions.push("Настроить платформенный SERP provider для конкурентных evidence, без ручной работы пользователя.");
|
||||
}
|
||||
|
||||
if (wordstat?.status !== "connected" && contract.readiness.seedCount > 0) {
|
||||
actions.push(`После настройки провайдера отправить ${contract.readiness.seedCount} seed-фраз в Wordstat queue.`);
|
||||
}
|
||||
|
||||
actions.push("Хранить внешние результаты как evidence при seed/brief/page, а не смешивать их с deterministic-анализом.");
|
||||
|
||||
return actions;
|
||||
}
|
||||
|
||||
export async function getMarketEnrichmentContract(projectId: string): Promise<MarketEnrichmentContract> {
|
||||
const analysis = await getLatestSemanticAnalysis(projectId);
|
||||
const providers = await buildProviderRegistry();
|
||||
const wordstat = await getLatestWordstatEvidence(projectId, analysis?.runId ?? null);
|
||||
const seedQueue = analysis ? buildSeedQueue(analysis, providers, wordstat) : [];
|
||||
const briefEvidence = analysis ? buildBriefEvidence(analysis, providers) : [];
|
||||
const configuredProviderCount = providers.filter((provider) => provider.status === "connected").length;
|
||||
const contractWithoutActions: Omit<MarketEnrichmentContract, "nextActions"> = {
|
||||
schemaVersion: "market-enrichment.v1",
|
||||
projectId,
|
||||
semanticRunId: analysis?.runId ?? null,
|
||||
generatedAt: new Date().toISOString(),
|
||||
state: getEnrichmentState(analysis, providers, seedQueue.length, wordstat),
|
||||
readiness: {
|
||||
canCollectWordstat:
|
||||
getProviderStatus(providers, "wordstat") === "connected" &&
|
||||
seedQueue.length > 0 &&
|
||||
wordstat.latestJob?.status !== "running",
|
||||
canCollectSerp: getProviderStatus(providers, "serp") === "connected" && briefEvidence.length > 0,
|
||||
seedCount: seedQueue.length,
|
||||
briefCount: briefEvidence.length,
|
||||
configuredProviderCount,
|
||||
userSetupRequired: providers.some((provider) => provider.userActionRequired)
|
||||
},
|
||||
providers,
|
||||
seedQueue,
|
||||
briefEvidence,
|
||||
wordstat
|
||||
};
|
||||
|
||||
return {
|
||||
...contractWithoutActions,
|
||||
nextActions: buildNextActions(contractWithoutActions)
|
||||
};
|
||||
}
|
||||
|
||||
export async function runMarketEnrichment(projectId: string): Promise<MarketEnrichmentContract> {
|
||||
const analysis = await getLatestSemanticAnalysis(projectId);
|
||||
const providers = await buildProviderRegistry();
|
||||
|
||||
if (!analysis || getProviderStatus(providers, "wordstat") !== "connected" || analysis.seedCandidates.length === 0) {
|
||||
return getMarketEnrichmentContract(projectId);
|
||||
}
|
||||
|
||||
try {
|
||||
await collectWordstatEvidence(projectId, analysis);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
return getMarketEnrichmentContract(projectId);
|
||||
}
|
||||
|
|
@ -0,0 +1,370 @@
|
|||
import { getWordstatRuntimeConfig, type WordstatRuntimeConfig } from "../settings/externalServiceSettings.js";
|
||||
|
||||
export type WordstatSeedInput = {
|
||||
seedId: string;
|
||||
phrase: string;
|
||||
clusterId: string;
|
||||
clusterTitle: string;
|
||||
priority: "high" | "medium" | "low";
|
||||
source: "detected" | "ontology";
|
||||
};
|
||||
|
||||
export type WordstatCollectRequest = {
|
||||
projectId: string;
|
||||
semanticRunId: string;
|
||||
region: string;
|
||||
seeds: WordstatSeedInput[];
|
||||
};
|
||||
|
||||
export type WordstatRelatedQuery = {
|
||||
phrase: string;
|
||||
frequency: number | null;
|
||||
frequencyGroup: string | null;
|
||||
raw: unknown;
|
||||
};
|
||||
|
||||
export type WordstatSeedResult = {
|
||||
seedId: string;
|
||||
seedPhrase: string;
|
||||
frequency: number | null;
|
||||
frequencyGroup: string | null;
|
||||
relatedQueries: WordstatRelatedQuery[];
|
||||
raw: unknown;
|
||||
};
|
||||
|
||||
export type WordstatCollectResult = {
|
||||
provider: "mcp_kv" | "yandex_api";
|
||||
mode: string;
|
||||
region: string;
|
||||
collectedAt: string;
|
||||
seedResults: WordstatSeedResult[];
|
||||
raw: unknown;
|
||||
};
|
||||
|
||||
export type WordstatProviderStatus =
|
||||
| {
|
||||
connected: true;
|
||||
provider: "mcp_kv" | "yandex_api";
|
||||
mode: string;
|
||||
message: string;
|
||||
}
|
||||
| {
|
||||
connected: false;
|
||||
provider: "disabled" | "mcp_kv" | "yandex_api";
|
||||
mode: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export interface WordstatProvider {
|
||||
getStatus(): WordstatProviderStatus;
|
||||
collect(request: WordstatCollectRequest): Promise<WordstatCollectResult>;
|
||||
}
|
||||
|
||||
export class WordstatProviderUnavailableError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "WordstatProviderUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
function getFrequencyGroup(frequency: number | null, explicitGroup?: unknown) {
|
||||
if (typeof explicitGroup === "string" && explicitGroup.trim()) {
|
||||
return explicitGroup.trim();
|
||||
}
|
||||
|
||||
if (frequency === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (frequency >= 10_000) {
|
||||
return "high";
|
||||
}
|
||||
|
||||
if (frequency >= 1_000) {
|
||||
return "mid";
|
||||
}
|
||||
|
||||
return "low";
|
||||
}
|
||||
|
||||
function getString(value: unknown) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function getNumber(value: unknown) {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return Math.max(0, Math.round(value));
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const normalized = Number(value.replace(/\s+/g, ""));
|
||||
|
||||
if (Number.isFinite(normalized)) {
|
||||
return Math.max(0, Math.round(normalized));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
|
||||
function asArray(value: unknown) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function normalizeRelatedQuery(value: unknown): WordstatRelatedQuery | null {
|
||||
const record = asRecord(value);
|
||||
const phrase =
|
||||
getString(record.phrase) ||
|
||||
getString(record.query) ||
|
||||
getString(record.keyword) ||
|
||||
getString(record.text) ||
|
||||
getString(record.name);
|
||||
|
||||
if (!phrase) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const frequency =
|
||||
getNumber(record.frequency) ?? getNumber(record.shows) ?? getNumber(record.count) ?? getNumber(record.impressions);
|
||||
|
||||
return {
|
||||
phrase,
|
||||
frequency,
|
||||
frequencyGroup: getFrequencyGroup(frequency, record.frequencyGroup ?? record.frequency_group ?? record.group),
|
||||
raw: value
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeSeedResult(value: unknown, seed: WordstatSeedInput): WordstatSeedResult {
|
||||
const record = asRecord(value);
|
||||
const phrase =
|
||||
getString(record.seedPhrase) ||
|
||||
getString(record.seed_phrase) ||
|
||||
getString(record.phrase) ||
|
||||
getString(record.query) ||
|
||||
seed.phrase;
|
||||
const frequency =
|
||||
getNumber(record.frequency) ??
|
||||
getNumber(record.shows) ??
|
||||
getNumber(record.count) ??
|
||||
getNumber(record.total) ??
|
||||
getNumber(record.impressions);
|
||||
const relatedSource =
|
||||
record.relatedQueries ??
|
||||
record.related_queries ??
|
||||
record.queries ??
|
||||
record.items ??
|
||||
record.results ??
|
||||
record.associations;
|
||||
|
||||
return {
|
||||
seedId: seed.seedId,
|
||||
seedPhrase: phrase,
|
||||
frequency,
|
||||
frequencyGroup: getFrequencyGroup(frequency, record.frequencyGroup ?? record.frequency_group ?? record.group),
|
||||
relatedQueries: asArray(relatedSource).flatMap((related) => {
|
||||
const normalized = normalizeRelatedQuery(related);
|
||||
return normalized ? [normalized] : [];
|
||||
}),
|
||||
raw: value
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeProviderResponse(
|
||||
provider: "mcp_kv" | "yandex_api",
|
||||
mode: string,
|
||||
region: string,
|
||||
seeds: WordstatSeedInput[],
|
||||
payload: unknown
|
||||
): WordstatCollectResult {
|
||||
const record = asRecord(payload);
|
||||
const resultSource = record.results ?? record.seedResults ?? record.seed_results ?? record.data ?? record.items;
|
||||
const resultRecords = asArray(resultSource);
|
||||
const resultByPhrase = new Map<string, unknown>();
|
||||
|
||||
for (const resultRecord of resultRecords) {
|
||||
const result = asRecord(resultRecord);
|
||||
const phrase =
|
||||
getString(result.seedPhrase) ||
|
||||
getString(result.seed_phrase) ||
|
||||
getString(result.phrase) ||
|
||||
getString(result.query) ||
|
||||
getString(result.keyword);
|
||||
|
||||
if (phrase) {
|
||||
resultByPhrase.set(phrase.toLowerCase(), resultRecord);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
provider,
|
||||
mode,
|
||||
region,
|
||||
collectedAt: new Date().toISOString(),
|
||||
seedResults: seeds.map((seed, index) =>
|
||||
normalizeSeedResult(resultByPhrase.get(seed.phrase.toLowerCase()) ?? resultRecords[index] ?? {}, seed)
|
||||
),
|
||||
raw: payload
|
||||
};
|
||||
}
|
||||
|
||||
class DisabledWordstatProvider implements WordstatProvider {
|
||||
getStatus(): WordstatProviderStatus {
|
||||
return {
|
||||
connected: false,
|
||||
provider: "disabled",
|
||||
mode: "disabled",
|
||||
message: "Wordstat provider отключён на платформе."
|
||||
};
|
||||
}
|
||||
|
||||
async collect(): Promise<WordstatCollectResult> {
|
||||
throw new WordstatProviderUnavailableError("Wordstat provider отключён на платформе.");
|
||||
}
|
||||
}
|
||||
|
||||
async function postJson(endpoint: string, body: unknown, headers: Record<string, string> = {}) {
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const message = await response.text().catch(() => "");
|
||||
throw new Error(`Wordstat provider вернул ${response.status}${message ? `: ${message.slice(0, 240)}` : ""}`);
|
||||
}
|
||||
|
||||
return response.json() as Promise<unknown>;
|
||||
}
|
||||
|
||||
class McpKvWordstatProvider implements WordstatProvider {
|
||||
constructor(private readonly endpoint: string | undefined) {}
|
||||
|
||||
getStatus(): WordstatProviderStatus {
|
||||
if (!this.endpoint) {
|
||||
return {
|
||||
connected: false,
|
||||
provider: "mcp_kv",
|
||||
mode: "mcp_kv",
|
||||
message: "WORDSTAT_MCP_ENDPOINT не настроен."
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
connected: true,
|
||||
provider: "mcp_kv",
|
||||
mode: "mcp_kv",
|
||||
message: "Wordstat MCP-KV endpoint настроен на платформе."
|
||||
};
|
||||
}
|
||||
|
||||
async collect(request: WordstatCollectRequest): Promise<WordstatCollectResult> {
|
||||
if (!this.endpoint) {
|
||||
throw new WordstatProviderUnavailableError("WORDSTAT_MCP_ENDPOINT не настроен.");
|
||||
}
|
||||
|
||||
const payload = await postJson(this.endpoint, {
|
||||
projectId: request.projectId,
|
||||
semanticRunId: request.semanticRunId,
|
||||
region: request.region,
|
||||
phrases: request.seeds.map((seed) => seed.phrase),
|
||||
seeds: request.seeds
|
||||
});
|
||||
|
||||
return normalizeProviderResponse("mcp_kv", "mcp_kv", request.region, request.seeds, payload);
|
||||
}
|
||||
}
|
||||
|
||||
class YandexApiWordstatProvider implements WordstatProvider {
|
||||
constructor(private readonly config: Extract<WordstatRuntimeConfig, { provider: "yandex_api" }> | null) {}
|
||||
|
||||
getStatus(): WordstatProviderStatus {
|
||||
if (!this.config) {
|
||||
return {
|
||||
connected: false,
|
||||
provider: "yandex_api",
|
||||
mode: "yandex_search_api",
|
||||
message: "Yandex Wordstat Search API не настроен: нужен folderId и API key или IAM token."
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
connected: true,
|
||||
provider: "yandex_api",
|
||||
mode: "yandex_search_api",
|
||||
message: "Yandex Wordstat Search API настроен на платформе."
|
||||
};
|
||||
}
|
||||
|
||||
async collect(request: WordstatCollectRequest): Promise<WordstatCollectResult> {
|
||||
if (!this.config) {
|
||||
throw new WordstatProviderUnavailableError("Yandex Wordstat Search API не настроен.");
|
||||
}
|
||||
|
||||
const endpoint = `${this.config.apiBaseUrl.replace(/\/+$/, "")}/topRequests`;
|
||||
const authorization =
|
||||
this.config.authType === "iam_token" ? `Bearer ${this.config.apiToken}` : `Api-Key ${this.config.apiToken}`;
|
||||
const seedResponses = [];
|
||||
|
||||
for (const seed of request.seeds) {
|
||||
const payload = await postJson(
|
||||
endpoint,
|
||||
{
|
||||
devices: this.config.devices,
|
||||
folderId: this.config.folderId,
|
||||
numPhrases: this.config.numPhrases,
|
||||
phrase: seed.phrase,
|
||||
regions: this.config.regionIds
|
||||
},
|
||||
{
|
||||
Authorization: authorization
|
||||
}
|
||||
);
|
||||
const record = asRecord(payload);
|
||||
const results = asArray(record.results ?? record.topRequests ?? record.items);
|
||||
const associations = asArray(record.associations ?? record.relatedQueries ?? record.related_queries);
|
||||
const exactMatch = results
|
||||
.map((item) => normalizeRelatedQuery(item))
|
||||
.find((item) => item?.phrase.toLowerCase() === seed.phrase.toLowerCase());
|
||||
|
||||
seedResponses.push({
|
||||
phrase: seed.phrase,
|
||||
frequency: exactMatch?.frequency ?? null,
|
||||
relatedQueries: [...results, ...associations],
|
||||
raw: payload
|
||||
});
|
||||
}
|
||||
|
||||
return normalizeProviderResponse(
|
||||
"yandex_api",
|
||||
"yandex_search_api",
|
||||
this.config.regionIds.join(","),
|
||||
request.seeds,
|
||||
{
|
||||
results: seedResponses
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createWordstatProvider(): Promise<WordstatProvider> {
|
||||
const config = await getWordstatRuntimeConfig();
|
||||
|
||||
if (config.provider === "mcp_kv") {
|
||||
return new McpKvWordstatProvider(config.mcpEndpoint);
|
||||
}
|
||||
|
||||
if (config.provider === "yandex_api") {
|
||||
return new YandexApiWordstatProvider(config);
|
||||
}
|
||||
|
||||
return new DisabledWordstatProvider();
|
||||
}
|
||||
|
|
@ -0,0 +1,693 @@
|
|||
import { pool } from "../db/client.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
import type { SemanticAnalysisRun } from "../analysis/semanticAnalysis.js";
|
||||
import {
|
||||
createWordstatProvider,
|
||||
type WordstatCollectResult,
|
||||
type WordstatSeedInput,
|
||||
WordstatProviderUnavailableError
|
||||
} from "./wordstatProvider.js";
|
||||
|
||||
type SeedRow = {
|
||||
id: string;
|
||||
phrase: string;
|
||||
cluster_id: string | null;
|
||||
cluster_title: string | null;
|
||||
priority: "high" | "medium" | "low" | null;
|
||||
source: "detected" | "ontology" | string;
|
||||
};
|
||||
|
||||
type WordstatJobRow = {
|
||||
id: string;
|
||||
project_id: string;
|
||||
semantic_run_id: string | null;
|
||||
status: "queued" | "running" | "done" | "failed" | "cancelled";
|
||||
provider: string;
|
||||
mode: string;
|
||||
region: string;
|
||||
requested_phrases: string[];
|
||||
raw_result_key: string | null;
|
||||
result_count: number;
|
||||
error_message: string | null;
|
||||
started_at: Date | null;
|
||||
completed_at: Date | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
type WordstatResultRow = {
|
||||
id: string;
|
||||
job_id: string;
|
||||
seed_id: string | null;
|
||||
phrase: string;
|
||||
source_phrase: string | null;
|
||||
normalized_phrase: string | null;
|
||||
frequency: number | null;
|
||||
frequency_group: string | null;
|
||||
region: string | null;
|
||||
position: number | null;
|
||||
raw: Record<string, unknown>;
|
||||
created_at: Date;
|
||||
};
|
||||
|
||||
export type WordstatEvidence = {
|
||||
latestJob: {
|
||||
id: string;
|
||||
status: WordstatJobRow["status"];
|
||||
provider: string;
|
||||
mode: string;
|
||||
region: string;
|
||||
requestedPhraseCount: number;
|
||||
rawResultKey: string | null;
|
||||
resultCount: number;
|
||||
errorMessage: string | null;
|
||||
startedAt: string | null;
|
||||
completedAt: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
} | null;
|
||||
seedResults: Array<{
|
||||
seedId: string | null;
|
||||
phrase: string;
|
||||
sourcePhrase: string | null;
|
||||
frequency: number | null;
|
||||
frequencyGroup: string | null;
|
||||
region: string | null;
|
||||
relatedCount: number;
|
||||
collectedAt: string;
|
||||
}>;
|
||||
topResults: Array<{
|
||||
phrase: string;
|
||||
sourcePhrase: string | null;
|
||||
frequency: number | null;
|
||||
frequencyGroup: string | null;
|
||||
region: string | null;
|
||||
}>;
|
||||
summary: {
|
||||
collectedSeedCount: number;
|
||||
exactFrequencySeedCount: number;
|
||||
relatedOnlySeedCount: number;
|
||||
noSignalSeedCount: number;
|
||||
resultCount: number;
|
||||
totalFrequency: number;
|
||||
highFrequencyCount: number;
|
||||
midFrequencyCount: number;
|
||||
lowFrequencyCount: number;
|
||||
};
|
||||
};
|
||||
|
||||
function toIsoDate(value: Date | null) {
|
||||
return value ? value.toISOString() : null;
|
||||
}
|
||||
|
||||
function normalizePhrase(value: string) {
|
||||
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
||||
}
|
||||
|
||||
const RELATED_QUERY_STOP_WORDS = new Set([
|
||||
"a",
|
||||
"an",
|
||||
"and",
|
||||
"by",
|
||||
"for",
|
||||
"in",
|
||||
"of",
|
||||
"on",
|
||||
"or",
|
||||
"the",
|
||||
"to",
|
||||
"без",
|
||||
"в",
|
||||
"для",
|
||||
"и",
|
||||
"или",
|
||||
"как",
|
||||
"на",
|
||||
"о",
|
||||
"об",
|
||||
"от",
|
||||
"по",
|
||||
"при",
|
||||
"с",
|
||||
"со",
|
||||
"что",
|
||||
"это"
|
||||
]);
|
||||
|
||||
const RELATED_QUERY_BLOCKLIST = [
|
||||
/\bbusiness\s+cats?\b/i,
|
||||
/\bcats?\b/i,
|
||||
/(^|\s)асу(\s|$)/i,
|
||||
/асу\s*тп/i,
|
||||
/расписан/i,
|
||||
/теори[яи]\s+автоматическ/i,
|
||||
/как\s+сделать\s+автоматическ\w*\s+содержан/i,
|
||||
/автоматическ\w*\s+содержан/i
|
||||
];
|
||||
|
||||
const PRODUCT_SIGNAL_PATTERNS = [
|
||||
/\b(ai|ии|bpm|crm|api|workflow)\b/i,
|
||||
/1с/i,
|
||||
/агент/i,
|
||||
/ассистент/i,
|
||||
/автоматизац/i,
|
||||
/бизнес/i,
|
||||
/данн/i,
|
||||
/закуп/i,
|
||||
/интеграц/i,
|
||||
/интеллект/i,
|
||||
/инфраструктур/i,
|
||||
/искусствен/i,
|
||||
/облак/i,
|
||||
/операц/i,
|
||||
/оркестр/i,
|
||||
/платформ/i,
|
||||
/процесс/i,
|
||||
/проект/i,
|
||||
/разработ/i,
|
||||
/роботизац/i,
|
||||
/тендер/i,
|
||||
/цифров/i,
|
||||
/bim/i,
|
||||
/on[-\s]?premise/i
|
||||
];
|
||||
|
||||
const AUTOMATION_SIGNAL_PATTERN = /автоматизац|роботизац|\bbpm\b|\bworkflow\b|процесс/i;
|
||||
|
||||
function stemToken(token: string) {
|
||||
if (/^[a-z0-9]+$/i.test(token)) {
|
||||
return token.toLowerCase();
|
||||
}
|
||||
|
||||
return token
|
||||
.toLocaleLowerCase("ru-RU")
|
||||
.replace(/(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i, "");
|
||||
}
|
||||
|
||||
function getMeaningfulTokenStems(phrase: string) {
|
||||
return new Set(
|
||||
phrase
|
||||
.toLocaleLowerCase("ru-RU")
|
||||
.replace(/ё/g, "е")
|
||||
.split(/[^a-zа-я0-9]+/i)
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length > 1 && !RELATED_QUERY_STOP_WORDS.has(token))
|
||||
.map(stemToken)
|
||||
.filter((token) => token.length > 1 && !RELATED_QUERY_STOP_WORDS.has(token))
|
||||
);
|
||||
}
|
||||
|
||||
function countIntersection(left: Set<string>, right: Set<string>) {
|
||||
let count = 0;
|
||||
|
||||
for (const item of left) {
|
||||
if (right.has(item)) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
function countProductSignals(phrase: string) {
|
||||
return PRODUCT_SIGNAL_PATTERNS.filter((pattern) => pattern.test(phrase)).length;
|
||||
}
|
||||
|
||||
function shouldKeepRelatedQuery(seedPhrase: string, relatedPhrase: string) {
|
||||
const normalizedSeedPhrase = normalizePhrase(seedPhrase);
|
||||
const normalizedRelatedPhrase = normalizePhrase(relatedPhrase);
|
||||
|
||||
if (!normalizedRelatedPhrase || normalizedRelatedPhrase === normalizedSeedPhrase) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (RELATED_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedRelatedPhrase))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const seedTokens = getMeaningfulTokenStems(normalizedSeedPhrase);
|
||||
const relatedTokens = getMeaningfulTokenStems(normalizedRelatedPhrase);
|
||||
const overlap = countIntersection(seedTokens, relatedTokens);
|
||||
const productSignals = countProductSignals(normalizedRelatedPhrase);
|
||||
const automationBridge =
|
||||
AUTOMATION_SIGNAL_PATTERN.test(normalizedSeedPhrase) && AUTOMATION_SIGNAL_PATTERN.test(normalizedRelatedPhrase);
|
||||
|
||||
if (overlap >= 2) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (overlap >= 1 && productSignals >= 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (automationBridge && productSignals >= 1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return productSignals >= 2;
|
||||
}
|
||||
|
||||
function mapJob(row: WordstatJobRow): NonNullable<WordstatEvidence["latestJob"]> {
|
||||
return {
|
||||
id: row.id,
|
||||
status: row.status,
|
||||
provider: row.provider,
|
||||
mode: row.mode,
|
||||
region: row.region,
|
||||
requestedPhraseCount: Array.isArray(row.requested_phrases) ? row.requested_phrases.length : 0,
|
||||
rawResultKey: row.raw_result_key,
|
||||
resultCount: row.result_count,
|
||||
errorMessage: row.error_message,
|
||||
startedAt: toIsoDate(row.started_at),
|
||||
completedAt: toIsoDate(row.completed_at),
|
||||
createdAt: row.created_at.toISOString(),
|
||||
updatedAt: row.updated_at.toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
function getFrequencyBucket(frequencyGroup: string | null) {
|
||||
const value = frequencyGroup?.toLowerCase();
|
||||
|
||||
if (!value) {
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
if (["high", "vch", "вч"].includes(value)) {
|
||||
return "high";
|
||||
}
|
||||
|
||||
if (["mid", "medium", "sch", "сч"].includes(value)) {
|
||||
return "mid";
|
||||
}
|
||||
|
||||
if (["low", "nch", "нч"].includes(value)) {
|
||||
return "low";
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]): WordstatEvidence {
|
||||
const seedRows = results.filter((result) => result.source_phrase === result.phrase || !result.source_phrase);
|
||||
const relatedCounts = new Map<string, number>();
|
||||
const topResultByPhrase = new Map<string, WordstatResultRow>();
|
||||
|
||||
for (const result of results) {
|
||||
if (result.source_phrase && result.source_phrase !== result.phrase) {
|
||||
relatedCounts.set(result.source_phrase, (relatedCounts.get(result.source_phrase) ?? 0) + 1);
|
||||
}
|
||||
|
||||
if (result.frequency !== null) {
|
||||
const normalizedPhrase = result.normalized_phrase ?? normalizePhrase(result.phrase);
|
||||
const currentResult = topResultByPhrase.get(normalizedPhrase);
|
||||
|
||||
if (!currentResult || (result.frequency ?? 0) > (currentResult.frequency ?? 0)) {
|
||||
topResultByPhrase.set(normalizedPhrase, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
latestJob: job ? mapJob(job) : null,
|
||||
seedResults: seedRows.map((result) => ({
|
||||
seedId: result.seed_id,
|
||||
phrase: result.phrase,
|
||||
sourcePhrase: result.source_phrase,
|
||||
frequency: result.frequency,
|
||||
frequencyGroup: result.frequency_group,
|
||||
region: result.region,
|
||||
relatedCount: relatedCounts.get(result.phrase) ?? 0,
|
||||
collectedAt: result.created_at.toISOString()
|
||||
})),
|
||||
topResults: Array.from(topResultByPhrase.values())
|
||||
.sort((left, right) => (right.frequency ?? 0) - (left.frequency ?? 0))
|
||||
.slice(0, 20)
|
||||
.map((result) => ({
|
||||
phrase: result.phrase,
|
||||
sourcePhrase: result.source_phrase,
|
||||
frequency: result.frequency,
|
||||
frequencyGroup: result.frequency_group,
|
||||
region: result.region
|
||||
})),
|
||||
summary: {
|
||||
collectedSeedCount: seedRows.length,
|
||||
exactFrequencySeedCount: seedRows.filter((result) => result.frequency !== null).length,
|
||||
relatedOnlySeedCount: seedRows.filter(
|
||||
(result) => result.frequency === null && (relatedCounts.get(result.phrase) ?? 0) > 0
|
||||
).length,
|
||||
noSignalSeedCount: seedRows.filter(
|
||||
(result) => result.frequency === null && (relatedCounts.get(result.phrase) ?? 0) === 0
|
||||
).length,
|
||||
resultCount: results.length,
|
||||
totalFrequency: results.reduce((sum, result) => sum + (result.frequency ?? 0), 0),
|
||||
highFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "high").length,
|
||||
midFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "mid").length,
|
||||
lowFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "low").length
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureSeedRows(projectId: string, analysis: SemanticAnalysisRun) {
|
||||
const seedRows: SeedRow[] = [];
|
||||
|
||||
for (const seed of analysis.seedCandidates.slice(0, 80)) {
|
||||
const inserted = await pool.query<SeedRow>(
|
||||
`
|
||||
insert into seeds (
|
||||
project_id,
|
||||
semantic_run_id,
|
||||
phrase,
|
||||
source,
|
||||
selected,
|
||||
reason,
|
||||
confidence,
|
||||
cluster_id,
|
||||
cluster_title,
|
||||
priority,
|
||||
updated_at
|
||||
)
|
||||
values ($1, $2, $3, $4, true, $5, $6, $7, $8, $9, now())
|
||||
on conflict do nothing
|
||||
returning id, phrase, cluster_id, cluster_title, priority, source;
|
||||
`,
|
||||
[
|
||||
projectId,
|
||||
analysis.runId,
|
||||
seed.phrase,
|
||||
seed.source,
|
||||
seed.reason,
|
||||
seed.priority === "high" ? 0.85 : seed.priority === "medium" ? 0.65 : 0.45,
|
||||
seed.clusterId,
|
||||
seed.clusterTitle,
|
||||
seed.priority
|
||||
]
|
||||
);
|
||||
|
||||
const row =
|
||||
inserted.rows[0] ??
|
||||
(
|
||||
await pool.query<SeedRow>(
|
||||
`
|
||||
select id, phrase, cluster_id, cluster_title, priority, source
|
||||
from seeds
|
||||
where project_id = $1
|
||||
and semantic_run_id = $2
|
||||
and lower(phrase) = lower($3)
|
||||
and coalesce(cluster_id, '') = coalesce($4, '')
|
||||
order by created_at asc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId, analysis.runId, seed.phrase, seed.clusterId]
|
||||
)
|
||||
).rows[0];
|
||||
|
||||
if (row) {
|
||||
seedRows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
return seedRows;
|
||||
}
|
||||
|
||||
function toWordstatSeed(row: SeedRow): WordstatSeedInput {
|
||||
return {
|
||||
seedId: row.id,
|
||||
phrase: row.phrase,
|
||||
clusterId: row.cluster_id ?? "",
|
||||
clusterTitle: row.cluster_title ?? "",
|
||||
priority: row.priority ?? "medium",
|
||||
source: row.source === "detected" ? "detected" : "ontology"
|
||||
};
|
||||
}
|
||||
|
||||
async function createJob(
|
||||
projectId: string,
|
||||
semanticRunId: string,
|
||||
provider: string,
|
||||
mode: string,
|
||||
region: string,
|
||||
seeds: WordstatSeedInput[]
|
||||
) {
|
||||
const result = await pool.query<WordstatJobRow>(
|
||||
`
|
||||
insert into wordstat_jobs (
|
||||
project_id,
|
||||
semantic_run_id,
|
||||
status,
|
||||
provider,
|
||||
mode,
|
||||
region,
|
||||
requested_phrases,
|
||||
started_at,
|
||||
updated_at
|
||||
)
|
||||
values ($1, $2, 'running', $3, $4, $5, $6, now(), now())
|
||||
returning
|
||||
id,
|
||||
project_id,
|
||||
semantic_run_id,
|
||||
status,
|
||||
provider,
|
||||
mode,
|
||||
region,
|
||||
requested_phrases,
|
||||
raw_result_key,
|
||||
result_count,
|
||||
error_message,
|
||||
started_at,
|
||||
completed_at,
|
||||
created_at,
|
||||
updated_at;
|
||||
`,
|
||||
[projectId, semanticRunId, provider, mode, region, JSON.stringify(seeds.map((seed) => seed.phrase))]
|
||||
);
|
||||
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function saveRawResult(projectId: string, semanticRunId: string, jobId: string, result: WordstatCollectResult) {
|
||||
const rawResultKey = `projects/${projectId}/wordstat/${semanticRunId}/${jobId}/raw.json`;
|
||||
|
||||
await storageProvider.putObject({
|
||||
key: rawResultKey,
|
||||
body: JSON.stringify(result.raw, null, 2),
|
||||
contentType: "application/json"
|
||||
});
|
||||
|
||||
return rawResultKey;
|
||||
}
|
||||
|
||||
async function saveResults(jobId: string, result: WordstatCollectResult) {
|
||||
let resultCount = 0;
|
||||
|
||||
for (const seedResult of result.seedResults) {
|
||||
await pool.query(
|
||||
`
|
||||
insert into wordstat_results (
|
||||
job_id,
|
||||
seed_id,
|
||||
phrase,
|
||||
source_phrase,
|
||||
normalized_phrase,
|
||||
frequency,
|
||||
frequency_group,
|
||||
region,
|
||||
position,
|
||||
raw
|
||||
)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, 0, $9);
|
||||
`,
|
||||
[
|
||||
jobId,
|
||||
seedResult.seedId,
|
||||
seedResult.seedPhrase,
|
||||
seedResult.seedPhrase,
|
||||
normalizePhrase(seedResult.seedPhrase),
|
||||
seedResult.frequency,
|
||||
seedResult.frequencyGroup,
|
||||
result.region,
|
||||
{
|
||||
source: "seed",
|
||||
provider: result.provider,
|
||||
mode: result.mode,
|
||||
collectedAt: result.collectedAt,
|
||||
raw: seedResult.raw
|
||||
}
|
||||
]
|
||||
);
|
||||
resultCount += 1;
|
||||
|
||||
const acceptedRelatedQueries = seedResult.relatedQueries.filter((relatedQuery) =>
|
||||
shouldKeepRelatedQuery(seedResult.seedPhrase, relatedQuery.phrase)
|
||||
);
|
||||
|
||||
for (const [index, relatedQuery] of acceptedRelatedQueries.entries()) {
|
||||
await pool.query(
|
||||
`
|
||||
insert into wordstat_results (
|
||||
job_id,
|
||||
seed_id,
|
||||
phrase,
|
||||
source_phrase,
|
||||
normalized_phrase,
|
||||
frequency,
|
||||
frequency_group,
|
||||
region,
|
||||
position,
|
||||
raw
|
||||
)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
|
||||
`,
|
||||
[
|
||||
jobId,
|
||||
seedResult.seedId,
|
||||
relatedQuery.phrase,
|
||||
seedResult.seedPhrase,
|
||||
normalizePhrase(relatedQuery.phrase),
|
||||
relatedQuery.frequency,
|
||||
relatedQuery.frequencyGroup,
|
||||
result.region,
|
||||
index + 1,
|
||||
{
|
||||
source: "related",
|
||||
provider: result.provider,
|
||||
mode: result.mode,
|
||||
collectedAt: result.collectedAt,
|
||||
raw: relatedQuery.raw
|
||||
}
|
||||
]
|
||||
);
|
||||
resultCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return resultCount;
|
||||
}
|
||||
|
||||
async function updateJobDone(jobId: string, rawResultKey: string, resultCount: number) {
|
||||
await pool.query(
|
||||
`
|
||||
update wordstat_jobs
|
||||
set status = 'done',
|
||||
raw_result_key = $2,
|
||||
result_count = $3,
|
||||
completed_at = now(),
|
||||
updated_at = now()
|
||||
where id = $1;
|
||||
`,
|
||||
[jobId, rawResultKey, resultCount]
|
||||
);
|
||||
}
|
||||
|
||||
async function updateJobFailed(jobId: string, error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "Не удалось собрать Wordstat evidence.";
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
update wordstat_jobs
|
||||
set status = 'failed',
|
||||
error_message = $2,
|
||||
completed_at = now(),
|
||||
updated_at = now()
|
||||
where id = $1;
|
||||
`,
|
||||
[jobId, message]
|
||||
);
|
||||
}
|
||||
|
||||
export async function collectWordstatEvidence(projectId: string, analysis: SemanticAnalysisRun) {
|
||||
const provider = await createWordstatProvider();
|
||||
const status = provider.getStatus();
|
||||
|
||||
if (!status.connected) {
|
||||
throw new WordstatProviderUnavailableError(status.message);
|
||||
}
|
||||
|
||||
const seedRows = await ensureSeedRows(projectId, analysis);
|
||||
const seeds = seedRows.map(toWordstatSeed);
|
||||
const region = "ru";
|
||||
const job = await createJob(projectId, analysis.runId, status.provider, status.mode, region, seeds);
|
||||
|
||||
if (!job) {
|
||||
throw new Error("Не удалось создать Wordstat job.");
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await provider.collect({
|
||||
projectId,
|
||||
semanticRunId: analysis.runId,
|
||||
region,
|
||||
seeds
|
||||
});
|
||||
const rawResultKey = await saveRawResult(projectId, analysis.runId, job.id, result);
|
||||
const resultCount = await saveResults(job.id, result);
|
||||
|
||||
await updateJobDone(job.id, rawResultKey, resultCount);
|
||||
} catch (error) {
|
||||
await updateJobFailed(job.id, error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
return getLatestWordstatEvidence(projectId, analysis.runId);
|
||||
}
|
||||
|
||||
export async function getLatestWordstatEvidence(projectId: string, semanticRunId: string | null): Promise<WordstatEvidence> {
|
||||
if (!semanticRunId) {
|
||||
return buildEvidence(null, []);
|
||||
}
|
||||
|
||||
const jobResult = await pool.query<WordstatJobRow>(
|
||||
`
|
||||
select
|
||||
id,
|
||||
project_id,
|
||||
semantic_run_id,
|
||||
status,
|
||||
provider,
|
||||
mode,
|
||||
region,
|
||||
requested_phrases,
|
||||
raw_result_key,
|
||||
result_count,
|
||||
error_message,
|
||||
started_at,
|
||||
completed_at,
|
||||
created_at,
|
||||
updated_at
|
||||
from wordstat_jobs
|
||||
where project_id = $1 and semantic_run_id = $2
|
||||
order by created_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId, semanticRunId]
|
||||
);
|
||||
const latestJob = jobResult.rows[0] ?? null;
|
||||
|
||||
if (!latestJob) {
|
||||
return buildEvidence(null, []);
|
||||
}
|
||||
|
||||
const results = await pool.query<WordstatResultRow>(
|
||||
`
|
||||
select
|
||||
id,
|
||||
job_id,
|
||||
seed_id,
|
||||
phrase,
|
||||
source_phrase,
|
||||
normalized_phrase,
|
||||
frequency,
|
||||
frequency_group,
|
||||
region,
|
||||
position,
|
||||
raw,
|
||||
created_at
|
||||
from wordstat_results
|
||||
where job_id = $1
|
||||
order by position asc, phrase asc;
|
||||
`,
|
||||
[latestJob.id]
|
||||
);
|
||||
|
||||
return buildEvidence(latestJob, results.rows);
|
||||
}
|
||||
|
|
@ -0,0 +1,859 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { Router } from "express";
|
||||
import type { Response } from "express";
|
||||
import { pool } from "../db/client.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
import { annotatePreviewTargets, getPreviewTargetSeeds } from "../workspace/previewTargets.js";
|
||||
import { getPreviewSnapshot } from "./snapshotCache.js";
|
||||
|
||||
type ProjectSourceRow = {
|
||||
type: string;
|
||||
config: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type PageSourceRow = {
|
||||
source_path: string;
|
||||
source_type: string;
|
||||
source_config: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type ResolvedAsset = {
|
||||
path: string;
|
||||
body: Uint8Array;
|
||||
contentType?: string;
|
||||
};
|
||||
|
||||
type PreviewRewriteOptions = {
|
||||
focusSelector?: string | null;
|
||||
focusIndex?: number | null;
|
||||
};
|
||||
|
||||
export const previewRouter = Router();
|
||||
|
||||
function getStringConfig(config: Record<string, unknown>, key: string) {
|
||||
const value = config[key];
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function getSourceConfig(source: ProjectSourceRow | PageSourceRow) {
|
||||
return "source_config" in source ? source.source_config : source.config;
|
||||
}
|
||||
|
||||
function getSourceType(source: ProjectSourceRow | PageSourceRow) {
|
||||
return "source_type" in source ? source.source_type : source.type;
|
||||
}
|
||||
|
||||
function safeDecode(value: string) {
|
||||
try {
|
||||
return decodeURIComponent(value);
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizePreviewPath(value: string) {
|
||||
const withoutQuery = safeDecode(value).split("?")[0]?.split("#")[0] ?? "";
|
||||
const normalized = path.posix.normalize(`/${withoutQuery.replace(/\\/g, "/")}`).replace(/^\/+/, "");
|
||||
|
||||
return normalized === "." ? "" : normalized;
|
||||
}
|
||||
|
||||
function encodePathname(value: string) {
|
||||
return value
|
||||
.split("/")
|
||||
.filter(Boolean)
|
||||
.map((part) => encodeURIComponent(part))
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function getMimeType(filePath: string) {
|
||||
const extension = path.extname(filePath).toLowerCase();
|
||||
const mimeTypes: Record<string, string> = {
|
||||
".avif": "image/avif",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".gif": "image/gif",
|
||||
".htm": "text/html; charset=utf-8",
|
||||
".html": "text/html; charset=utf-8",
|
||||
".ico": "image/x-icon",
|
||||
".jpeg": "image/jpeg",
|
||||
".jpg": "image/jpeg",
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".map": "application/json; charset=utf-8",
|
||||
".mp4": "video/mp4",
|
||||
".otf": "font/otf",
|
||||
".png": "image/png",
|
||||
".svg": "image/svg+xml; charset=utf-8",
|
||||
".ttf": "font/ttf",
|
||||
".txt": "text/plain; charset=utf-8",
|
||||
".webm": "video/webm",
|
||||
".webmanifest": "application/manifest+json; charset=utf-8",
|
||||
".webp": "image/webp",
|
||||
".woff": "font/woff",
|
||||
".woff2": "font/woff2",
|
||||
".xml": "application/xml; charset=utf-8"
|
||||
};
|
||||
|
||||
return mimeTypes[extension] ?? "application/octet-stream";
|
||||
}
|
||||
|
||||
function isHtml(filePath: string, contentType?: string) {
|
||||
return /\.html?$/i.test(filePath) || Boolean(contentType?.includes("text/html"));
|
||||
}
|
||||
|
||||
function isCss(filePath: string, contentType?: string) {
|
||||
return /\.css$/i.test(filePath) || Boolean(contentType?.includes("text/css"));
|
||||
}
|
||||
|
||||
function isTextLike(filePath: string, contentType?: string) {
|
||||
return isHtml(filePath, contentType) || isCss(filePath, contentType) || /\.(js|json|svg|txt|xml|webmanifest)$/i.test(filePath);
|
||||
}
|
||||
|
||||
function getSiteRoot(config: Record<string, unknown>, currentPath: string) {
|
||||
const rootName = getStringConfig(config, "rootName");
|
||||
|
||||
if (rootName) {
|
||||
const normalizedRoot = normalizePreviewPath(rootName);
|
||||
|
||||
if (normalizedRoot && (currentPath === normalizedRoot || currentPath.startsWith(`${normalizedRoot}/`))) {
|
||||
return normalizedRoot;
|
||||
}
|
||||
}
|
||||
|
||||
const firstSegment = currentPath.split("/").filter(Boolean)[0] ?? "";
|
||||
|
||||
return firstSegment.includes(".") ? "" : firstSegment;
|
||||
}
|
||||
|
||||
function buildCandidatePaths(requestedPath: string, config: Record<string, unknown>) {
|
||||
const normalizedPath = normalizePreviewPath(requestedPath);
|
||||
const rootName = normalizePreviewPath(getStringConfig(config, "rootName") ?? "");
|
||||
const candidates = new Set<string>();
|
||||
|
||||
function addCandidate(candidate: string) {
|
||||
const normalized = normalizePreviewPath(candidate);
|
||||
|
||||
if (!normalized) {
|
||||
candidates.add("index.html");
|
||||
return;
|
||||
}
|
||||
|
||||
candidates.add(normalized);
|
||||
|
||||
if (normalized.endsWith("/")) {
|
||||
candidates.add(`${normalized}index.html`);
|
||||
}
|
||||
}
|
||||
|
||||
addCandidate(normalizedPath);
|
||||
|
||||
if (rootName) {
|
||||
if (!normalizedPath.startsWith(`${rootName}/`) && normalizedPath !== rootName) {
|
||||
addCandidate(`${rootName}/${normalizedPath}`);
|
||||
}
|
||||
|
||||
if (normalizedPath.startsWith(`${rootName}/`)) {
|
||||
addCandidate(normalizedPath.slice(rootName.length + 1));
|
||||
}
|
||||
}
|
||||
|
||||
return [...candidates];
|
||||
}
|
||||
|
||||
async function getLatestProjectSource(projectId: string) {
|
||||
const result = await pool.query<ProjectSourceRow>(
|
||||
`
|
||||
select type, config
|
||||
from project_sources
|
||||
where project_id = $1
|
||||
order by created_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function getLatestPageSource(projectId: string, pageId: string) {
|
||||
const result = await pool.query<PageSourceRow>(
|
||||
`
|
||||
select pv.source_path, ps.type as source_type, ps.config as source_config
|
||||
from page_versions pv
|
||||
join scan_versions sv on sv.id = pv.scan_version_id
|
||||
join project_sources ps on ps.id = sv.source_id
|
||||
where sv.project_id = $1
|
||||
and pv.page_id = $2
|
||||
and sv.status = 'done'
|
||||
order by sv.completed_at desc nulls last, sv.created_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId, pageId]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function readBrowserFolderAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
|
||||
const sourceConfig = getSourceConfig(source);
|
||||
const importPrefix = getStringConfig(sourceConfig, "importPrefix");
|
||||
|
||||
if (!importPrefix) {
|
||||
throw new Error("У проекта нет importPrefix.");
|
||||
}
|
||||
|
||||
const normalizedPrefix = importPrefix.endsWith("/") ? importPrefix : `${importPrefix}/`;
|
||||
|
||||
for (const candidate of buildCandidatePaths(requestedPath, sourceConfig)) {
|
||||
try {
|
||||
const object = await storageProvider.getObject(`${normalizedPrefix}${candidate}`);
|
||||
|
||||
return {
|
||||
path: candidate,
|
||||
body: object.body,
|
||||
contentType: object.contentType
|
||||
};
|
||||
} catch {
|
||||
// Try the next deterministic candidate: rootName/path, path without rootName, etc.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function readLocalFolderAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
|
||||
const sourceConfig = getSourceConfig(source);
|
||||
const sourceRoot = getStringConfig(sourceConfig, "path");
|
||||
|
||||
if (!sourceRoot) {
|
||||
throw new Error("У проекта нет локального пути.");
|
||||
}
|
||||
|
||||
const resolvedRoot = path.resolve(sourceRoot);
|
||||
|
||||
for (const candidate of buildCandidatePaths(requestedPath, sourceConfig)) {
|
||||
const resolvedPath = path.resolve(resolvedRoot, candidate);
|
||||
|
||||
if (resolvedPath !== resolvedRoot && !resolvedPath.startsWith(`${resolvedRoot}${path.sep}`)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
return {
|
||||
path: candidate,
|
||||
body: await fs.readFile(resolvedPath),
|
||||
contentType: getMimeType(candidate)
|
||||
};
|
||||
} catch {
|
||||
// Try the next deterministic candidate.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function readProjectAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
|
||||
if (getSourceType(source) === "browser_folder") {
|
||||
return readBrowserFolderAsset(source, requestedPath);
|
||||
}
|
||||
|
||||
if (getSourceType(source) === "local_folder") {
|
||||
return readLocalFolderAsset(source, requestedPath);
|
||||
}
|
||||
|
||||
throw new Error("Preview для этого типа источника пока не поддерживается.");
|
||||
}
|
||||
|
||||
function sendNotFound(response: Response) {
|
||||
response.status(404).type("text/plain").send("Preview asset not found.");
|
||||
}
|
||||
|
||||
function splitUrlSuffix(value: string) {
|
||||
const suffixIndex = value.search(/[?#]/);
|
||||
|
||||
if (suffixIndex === -1) {
|
||||
return { pathname: value, suffix: "" };
|
||||
}
|
||||
|
||||
return {
|
||||
pathname: value.slice(0, suffixIndex),
|
||||
suffix: value.slice(suffixIndex)
|
||||
};
|
||||
}
|
||||
|
||||
function shouldKeepOriginalUrl(value: string) {
|
||||
const trimmed = value.trim();
|
||||
|
||||
return (
|
||||
!trimmed ||
|
||||
trimmed.startsWith("#") ||
|
||||
trimmed.startsWith("//") ||
|
||||
/^[a-z][a-z0-9+.-]*:/i.test(trimmed)
|
||||
);
|
||||
}
|
||||
|
||||
function buildPreviewSiteUrl(projectId: string, assetPath: string) {
|
||||
return `/preview/projects/${encodeURIComponent(projectId)}/site/${encodePathname(assetPath)}`;
|
||||
}
|
||||
|
||||
function buildPreviewDirectoryUrl(projectId: string, directoryPath: string) {
|
||||
const normalizedDirectory = normalizePreviewPath(directoryPath);
|
||||
const siteUrl = buildPreviewSiteUrl(projectId, normalizedDirectory);
|
||||
|
||||
return siteUrl.endsWith("/") ? siteUrl : `${siteUrl}/`;
|
||||
}
|
||||
|
||||
function rewritePreviewUrl(value: string, projectId: string, siteRoot: string) {
|
||||
const trimmed = value.trim();
|
||||
|
||||
if (shouldKeepOriginalUrl(trimmed) || !trimmed.startsWith("/") || trimmed.startsWith("//")) {
|
||||
return value;
|
||||
}
|
||||
|
||||
const { pathname, suffix } = splitUrlSuffix(trimmed);
|
||||
const rootedPath = normalizePreviewPath(`${siteRoot}/${pathname.replace(/^\/+/, "")}`);
|
||||
|
||||
return `${buildPreviewSiteUrl(projectId, rootedPath)}${suffix}`;
|
||||
}
|
||||
|
||||
function rewriteSrcset(value: string, projectId: string, siteRoot: string) {
|
||||
return value
|
||||
.split(",")
|
||||
.map((entry) => {
|
||||
const trimmedEntry = entry.trim();
|
||||
|
||||
if (!trimmedEntry) {
|
||||
return entry;
|
||||
}
|
||||
|
||||
const [url, ...descriptor] = trimmedEntry.split(/\s+/);
|
||||
return [rewritePreviewUrl(url ?? "", projectId, siteRoot), ...descriptor].join(" ");
|
||||
})
|
||||
.join(", ");
|
||||
}
|
||||
|
||||
function buildPreviewBridgeScript() {
|
||||
return `
|
||||
<script>
|
||||
(() => {
|
||||
const state = { workspaceKey: "", targets: [] };
|
||||
const layerId = "seo-mode-preview-marker-layer";
|
||||
let frame = 0;
|
||||
let lastActivation = { id: "", at: 0 };
|
||||
|
||||
function ensureLayer() {
|
||||
let layer = document.getElementById(layerId);
|
||||
|
||||
if (layer) {
|
||||
return layer;
|
||||
}
|
||||
|
||||
layer = document.createElement("div");
|
||||
layer.id = layerId;
|
||||
layer.setAttribute("aria-hidden", "true");
|
||||
layer.style.position = "absolute";
|
||||
layer.style.left = "0";
|
||||
layer.style.top = "0";
|
||||
layer.style.zIndex = "2147483647";
|
||||
layer.style.pointerEvents = "none";
|
||||
document.body.appendChild(layer);
|
||||
return layer;
|
||||
}
|
||||
|
||||
function scheduleMeasure() {
|
||||
window.cancelAnimationFrame(frame);
|
||||
frame = window.requestAnimationFrame(measure);
|
||||
}
|
||||
|
||||
function syncLayerBounds(layer) {
|
||||
const root = document.documentElement;
|
||||
const width = Math.max(root.scrollWidth, document.body?.scrollWidth || 0, window.innerWidth);
|
||||
const height = Math.max(root.scrollHeight, document.body?.scrollHeight || 0, window.innerHeight);
|
||||
layer.style.width = width + "px";
|
||||
layer.style.height = height + "px";
|
||||
}
|
||||
|
||||
function intersects(rect) {
|
||||
return rect.right > rect.left && rect.bottom > rect.top;
|
||||
}
|
||||
|
||||
function intersectRects(rect, clipRect) {
|
||||
return {
|
||||
left: Math.max(rect.left, clipRect.left),
|
||||
top: Math.max(rect.top, clipRect.top),
|
||||
right: Math.min(rect.right, clipRect.right),
|
||||
bottom: Math.min(rect.bottom, clipRect.bottom)
|
||||
};
|
||||
}
|
||||
|
||||
function getVisibleRect(element) {
|
||||
let visibleRect = element.getBoundingClientRect();
|
||||
|
||||
if (visibleRect.width < 2 || visibleRect.height < 2) return null;
|
||||
|
||||
visibleRect = {
|
||||
left: visibleRect.left,
|
||||
top: visibleRect.top,
|
||||
right: visibleRect.right,
|
||||
bottom: visibleRect.bottom
|
||||
};
|
||||
|
||||
visibleRect = intersectRects(visibleRect, {
|
||||
left: 0,
|
||||
top: 0,
|
||||
right: window.innerWidth,
|
||||
bottom: window.innerHeight
|
||||
});
|
||||
|
||||
if (!intersects(visibleRect)) return null;
|
||||
|
||||
let parent = element.parentElement;
|
||||
|
||||
while (parent && parent !== document.body && parent !== document.documentElement) {
|
||||
const parentStyle = window.getComputedStyle(parent);
|
||||
const clipsContent = /(auto|scroll|hidden|clip)/.test(
|
||||
[parentStyle.overflow, parentStyle.overflowX, parentStyle.overflowY].join(" ")
|
||||
);
|
||||
|
||||
if (clipsContent) {
|
||||
visibleRect = intersectRects(visibleRect, parent.getBoundingClientRect());
|
||||
|
||||
if (!intersects(visibleRect)) return null;
|
||||
}
|
||||
|
||||
parent = parent.parentElement;
|
||||
}
|
||||
|
||||
return visibleRect;
|
||||
}
|
||||
|
||||
function getUsableRect(selector) {
|
||||
if (!selector) return null;
|
||||
const element = document.querySelector(selector);
|
||||
if (!element) return null;
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return null;
|
||||
return getVisibleRect(element);
|
||||
}
|
||||
|
||||
function getHorizontalScroller() {
|
||||
const candidates = Array.from(document.querySelectorAll("*"))
|
||||
.filter((element) => element instanceof HTMLElement)
|
||||
.filter((element) => element.scrollWidth > element.clientWidth + 8)
|
||||
.map((element) => {
|
||||
const rect = element.getBoundingClientRect();
|
||||
const overflow = element.scrollWidth - element.clientWidth;
|
||||
const area = Math.max(0, rect.width) * Math.max(0, rect.height);
|
||||
|
||||
return { element, overflow, area };
|
||||
})
|
||||
.filter((candidate) => candidate.area > 1000);
|
||||
|
||||
candidates.sort((first, second) => {
|
||||
if (second.overflow !== first.overflow) return second.overflow - first.overflow;
|
||||
return second.area - first.area;
|
||||
});
|
||||
|
||||
return candidates[0]?.element ?? null;
|
||||
}
|
||||
|
||||
function getPageStartPosition() {
|
||||
const scroller = getHorizontalScroller();
|
||||
|
||||
if (scroller) {
|
||||
const rect = scroller.getBoundingClientRect();
|
||||
|
||||
return {
|
||||
left: rect.left + window.scrollX - scroller.scrollLeft + 14,
|
||||
top: rect.top + window.scrollY + 14
|
||||
};
|
||||
}
|
||||
|
||||
return { left: 14, top: 14 };
|
||||
}
|
||||
|
||||
function activateTarget(target, event) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
|
||||
if (typeof event.stopImmediatePropagation === "function") {
|
||||
event.stopImmediatePropagation();
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
|
||||
if (lastActivation.id === target.id && now - lastActivation.at < 250) {
|
||||
return;
|
||||
}
|
||||
|
||||
lastActivation = { id: target.id, at: now };
|
||||
|
||||
window.parent?.postMessage({
|
||||
type: "seo-mode:activate-target",
|
||||
workspaceKey: state.workspaceKey,
|
||||
targetId: target.id,
|
||||
sectionId: target.sectionId || ""
|
||||
}, "*");
|
||||
}
|
||||
|
||||
function createMarker(target, left, top) {
|
||||
const marker = document.createElement("button");
|
||||
marker.type = "button";
|
||||
marker.textContent = String(target.number);
|
||||
marker.title = target.label || "Открыть поле правки";
|
||||
marker.style.position = "absolute";
|
||||
marker.style.left = left + "px";
|
||||
marker.style.top = top + "px";
|
||||
marker.style.width = "30px";
|
||||
marker.style.height = "30px";
|
||||
marker.style.display = "grid";
|
||||
marker.style.placeItems = "center";
|
||||
marker.style.padding = "0";
|
||||
marker.style.color = "#fff";
|
||||
marker.style.background = "#f27a1a";
|
||||
marker.style.border = "2px solid rgba(255,255,255,.95)";
|
||||
marker.style.borderRadius = "999px";
|
||||
marker.style.boxShadow = "0 6px 16px rgba(242,122,26,.28)";
|
||||
marker.style.cursor = "pointer";
|
||||
marker.style.font = "700 12px/1 Inter, Arial, sans-serif";
|
||||
marker.style.pointerEvents = "auto";
|
||||
marker.style.touchAction = "manipulation";
|
||||
marker.style.userSelect = "none";
|
||||
marker.addEventListener("pointerdown", (event) => activateTarget(target, event));
|
||||
marker.addEventListener("click", (event) => activateTarget(target, event));
|
||||
return marker;
|
||||
}
|
||||
|
||||
function measure() {
|
||||
const layer = ensureLayer();
|
||||
const root = document.documentElement;
|
||||
const documentWidth = Math.max(root.scrollWidth, document.body?.scrollWidth || 0, window.innerWidth);
|
||||
syncLayerBounds(layer);
|
||||
layer.textContent = "";
|
||||
const clusters = new Map();
|
||||
|
||||
for (const target of state.targets) {
|
||||
try {
|
||||
const exactSelectors = Array.isArray(target.selectors) && target.selectors.length > 0
|
||||
? target.selectors
|
||||
: [];
|
||||
const prefersPageStart = target.preferPageStart || (target.pageStartFallback && exactSelectors.length === 0);
|
||||
const selectors = prefersPageStart ? [] : (exactSelectors.length > 0 ? exactSelectors : [target.selector].filter(Boolean));
|
||||
let placedExactMarker = false;
|
||||
let placedFallbackMarker = false;
|
||||
|
||||
for (const selector of selectors) {
|
||||
const rect = getUsableRect(selector);
|
||||
if (!rect) continue;
|
||||
placedExactMarker = true;
|
||||
const baseLeft = Math.max(12, rect.left + window.scrollX + 10);
|
||||
const baseTop = Math.max(12, rect.top + window.scrollY + 10);
|
||||
const clusterKey = Math.round(baseLeft / 40) + ":" + Math.round(baseTop / 40);
|
||||
const cluster = clusters.get(clusterKey) || [];
|
||||
cluster.push({ target, baseLeft, baseTop });
|
||||
clusters.set(clusterKey, cluster);
|
||||
}
|
||||
|
||||
if (!prefersPageStart && !placedExactMarker && target.fallbackSelector) {
|
||||
const rect = getUsableRect(target.fallbackSelector);
|
||||
if (rect) {
|
||||
placedFallbackMarker = true;
|
||||
const baseLeft = Math.max(12, rect.left + window.scrollX + 10);
|
||||
const baseTop = Math.max(12, rect.top + window.scrollY + 10);
|
||||
const clusterKey = Math.round(baseLeft / 40) + ":" + Math.round(baseTop / 40);
|
||||
const cluster = clusters.get(clusterKey) || [];
|
||||
cluster.push({ target, baseLeft, baseTop });
|
||||
clusters.set(clusterKey, cluster);
|
||||
}
|
||||
}
|
||||
|
||||
if (!placedExactMarker && !placedFallbackMarker && target.pageStartFallback) {
|
||||
const pageStartPosition = getPageStartPosition();
|
||||
const baseLeft = pageStartPosition.left;
|
||||
const baseTop = pageStartPosition.top;
|
||||
const clusterKey = "page-start";
|
||||
const cluster = clusters.get(clusterKey) || [];
|
||||
cluster.push({ target, baseLeft, baseTop, allowViewportOverflow: true });
|
||||
clusters.set(clusterKey, cluster);
|
||||
}
|
||||
} catch {
|
||||
// A broken selector should not break the preview bridge.
|
||||
}
|
||||
}
|
||||
|
||||
for (const cluster of clusters.values()) {
|
||||
const markerSize = 30;
|
||||
const gap = 6;
|
||||
const maxColumns = 8;
|
||||
const columns = Math.min(maxColumns, cluster.length);
|
||||
const clusterWidth = columns * markerSize + Math.max(0, columns - 1) * gap;
|
||||
const shouldClamp = !cluster.some((entry) => entry.allowViewportOverflow);
|
||||
const startLeft = shouldClamp
|
||||
? Math.min(
|
||||
Math.max(12, cluster[0].baseLeft),
|
||||
Math.max(12, documentWidth - clusterWidth - 12)
|
||||
)
|
||||
: cluster[0].baseLeft;
|
||||
const startTop = Math.max(12, cluster[0].baseTop);
|
||||
|
||||
cluster.forEach((entry, index) => {
|
||||
const column = index % columns;
|
||||
const row = Math.floor(index / columns);
|
||||
layer.appendChild(
|
||||
createMarker(
|
||||
entry.target,
|
||||
startLeft + column * (markerSize + gap),
|
||||
startTop + row * (markerSize + gap)
|
||||
)
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener("message", (event) => {
|
||||
const data = event.data;
|
||||
|
||||
if (!data || data.type !== "seo-mode:measure-targets") {
|
||||
return;
|
||||
}
|
||||
|
||||
state.workspaceKey = data.workspaceKey || "";
|
||||
state.targets = Array.isArray(data.targets) ? data.targets : [];
|
||||
scheduleMeasure();
|
||||
});
|
||||
|
||||
window.addEventListener("resize", scheduleMeasure);
|
||||
window.addEventListener("scroll", scheduleMeasure, { passive: true });
|
||||
document.addEventListener("scroll", scheduleMeasure, { capture: true, passive: true });
|
||||
window.addEventListener("load", () => {
|
||||
window.setTimeout(scheduleMeasure, 50);
|
||||
window.setTimeout(scheduleMeasure, 500);
|
||||
window.setTimeout(scheduleMeasure, 1500);
|
||||
});
|
||||
document.addEventListener("DOMContentLoaded", () => {
|
||||
window.setTimeout(scheduleMeasure, 50);
|
||||
window.setTimeout(scheduleMeasure, 500);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
`;
|
||||
}
|
||||
|
||||
function buildPreviewFocusScript(selector: string | null | undefined, index: number | null | undefined = null) {
|
||||
if (!selector) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return `
|
||||
<script>
|
||||
(() => {
|
||||
const selector = ${JSON.stringify(selector)};
|
||||
const targetIndex = ${Number.isInteger(index) && index !== null ? index : 0};
|
||||
const focusTarget = () => {
|
||||
let target = null;
|
||||
|
||||
try {
|
||||
const matches = Array.from(document.querySelectorAll(selector));
|
||||
target = matches[Math.max(0, Math.min(targetIndex, matches.length - 1))] || null;
|
||||
} catch {
|
||||
target = null;
|
||||
}
|
||||
|
||||
if (!target) return;
|
||||
|
||||
target.scrollIntoView({ block: "center", inline: "center" });
|
||||
|
||||
if (target instanceof HTMLElement) {
|
||||
target.style.outline = "2px solid rgba(34, 197, 94, 0.55)";
|
||||
target.style.outlineOffset = "8px";
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener("load", () => {
|
||||
window.setTimeout(focusTarget, 80);
|
||||
window.setTimeout(focusTarget, 600);
|
||||
window.setTimeout(focusTarget, 1400);
|
||||
});
|
||||
document.addEventListener("DOMContentLoaded", () => window.setTimeout(focusTarget, 80));
|
||||
})();
|
||||
</script>
|
||||
`;
|
||||
}
|
||||
|
||||
function getQueryString(value: unknown) {
|
||||
const candidate = Array.isArray(value) ? value[0] : value;
|
||||
|
||||
if (typeof candidate !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trimmed = candidate.trim();
|
||||
|
||||
return trimmed.length > 0 && trimmed.length < 300 ? trimmed : null;
|
||||
}
|
||||
|
||||
function getQueryInteger(value: unknown) {
|
||||
const candidate = Array.isArray(value) ? value[0] : value;
|
||||
const parsed = typeof candidate === "string" ? Number.parseInt(candidate, 10) : Number.NaN;
|
||||
|
||||
return Number.isInteger(parsed) && parsed >= 0 && parsed < 200 ? parsed : null;
|
||||
}
|
||||
|
||||
function getQueryDimension(value: unknown, fallback: number) {
|
||||
const candidate = Array.isArray(value) ? value[0] : value;
|
||||
const parsed = typeof candidate === "string" ? Number.parseInt(candidate, 10) : Number.NaN;
|
||||
|
||||
return Number.isInteger(parsed) && parsed >= 320 && parsed <= 4096 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function rewriteHtml(
|
||||
html: string,
|
||||
projectId: string,
|
||||
resolvedPath: string,
|
||||
sourceConfig: Record<string, unknown>,
|
||||
options: PreviewRewriteOptions = {}
|
||||
) {
|
||||
const siteRoot = getSiteRoot(sourceConfig, resolvedPath);
|
||||
const currentDirectory = path.posix.dirname(resolvedPath);
|
||||
const baseDirectory = currentDirectory === "." ? "" : `${currentDirectory}/`;
|
||||
const baseHref = buildPreviewDirectoryUrl(projectId, baseDirectory);
|
||||
const annotatedHtml = annotatePreviewTargets(html, getPreviewTargetSeeds(html));
|
||||
const withoutBase = annotatedHtml.replace(/<base\b[^>]*>/gi, "");
|
||||
const withRootedAttributes = withoutBase
|
||||
.replace(/\s(src|href|poster|action)\s*=\s*(["'])([^"']*)\2/gi, (match, attribute: string, quote: string, value: string) => {
|
||||
return ` ${attribute}=${quote}${rewritePreviewUrl(value, projectId, siteRoot)}${quote}`;
|
||||
})
|
||||
.replace(/\s(srcset)\s*=\s*(["'])([^"']*)\2/gi, (match, attribute: string, quote: string, value: string) => {
|
||||
return ` ${attribute}=${quote}${rewriteSrcset(value, projectId, siteRoot)}${quote}`;
|
||||
});
|
||||
const baseTag = `<base href="${baseHref}">`;
|
||||
const bridgeScript = buildPreviewBridgeScript();
|
||||
const focusScript = buildPreviewFocusScript(options.focusSelector, options.focusIndex);
|
||||
|
||||
if (/<head\b[^>]*>/i.test(withRootedAttributes)) {
|
||||
return withRootedAttributes
|
||||
.replace(/<head\b[^>]*>/i, (headTag) => `${headTag}${baseTag}`)
|
||||
.replace(/<\/body>/i, `${bridgeScript}${focusScript}</body>`);
|
||||
}
|
||||
|
||||
return `<!doctype html><html><head>${baseTag}</head><body>${withRootedAttributes}${bridgeScript}${focusScript}</body></html>`;
|
||||
}
|
||||
|
||||
function rewriteCss(css: string, projectId: string, resolvedPath: string, sourceConfig: Record<string, unknown>) {
|
||||
const siteRoot = getSiteRoot(sourceConfig, resolvedPath);
|
||||
|
||||
return css
|
||||
.replace(/url\(\s*(["']?)([^"')]+)\1\s*\)/gi, (_match, quote: string, value: string) => {
|
||||
const rewritten = rewritePreviewUrl(value, projectId, siteRoot);
|
||||
return `url(${quote}${rewritten}${quote})`;
|
||||
})
|
||||
.replace(/@import\s+(["'])([^"']+)\1/gi, (_match, quote: string, value: string) => {
|
||||
return `@import ${quote}${rewritePreviewUrl(value, projectId, siteRoot)}${quote}`;
|
||||
});
|
||||
}
|
||||
|
||||
function textFromAsset(asset: ResolvedAsset) {
|
||||
return new TextDecoder().decode(asset.body);
|
||||
}
|
||||
|
||||
function sendAsset(response: Response, asset: ResolvedAsset, body: string | Uint8Array, contentType?: string) {
|
||||
response.setHeader("Cache-Control", "no-store");
|
||||
response.type(contentType ?? asset.contentType ?? getMimeType(asset.path));
|
||||
response.send(typeof body === "string" ? body : Buffer.from(body));
|
||||
}
|
||||
|
||||
previewRouter.get("/projects/:projectId/pages/:pageId", async (request, response) => {
|
||||
try {
|
||||
const page = await getLatestPageSource(request.params.projectId, request.params.pageId);
|
||||
|
||||
if (!page) {
|
||||
sendNotFound(response);
|
||||
return;
|
||||
}
|
||||
|
||||
response.redirect(302, buildPreviewSiteUrl(request.params.projectId, page.source_path));
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
response.status(500).type("text/plain").send("Preview page failed.");
|
||||
}
|
||||
});
|
||||
|
||||
previewRouter.get(/^\/projects\/([^/]+)\/snapshot\/?(.*)$/i, async (request, response) => {
|
||||
try {
|
||||
const params = request.params as Record<string, string | undefined>;
|
||||
const projectId = params[0] ?? "";
|
||||
const rawAssetPath = params[1] ?? "";
|
||||
const cacheVersion = getQueryString(request.query.cacheVersion) ?? getQueryString(request.query.scanId) ?? "latest";
|
||||
const focusSelector = getQueryString(request.query.focus);
|
||||
const focusIndex = getQueryInteger(request.query.focusIndex);
|
||||
const viewportWidth = getQueryDimension(request.query.viewportWidth, 1920);
|
||||
const viewportHeight = getQueryDimension(request.query.viewportHeight, 1080);
|
||||
const origin = `${request.protocol}://${request.get("host")}`;
|
||||
const snapshot = await getPreviewSnapshot({
|
||||
focusIndex,
|
||||
focusSelector,
|
||||
origin,
|
||||
projectId,
|
||||
cacheVersion,
|
||||
sourcePath: rawAssetPath || "index.html",
|
||||
viewportHeight,
|
||||
viewportWidth
|
||||
});
|
||||
|
||||
response
|
||||
.status(200)
|
||||
.setHeader("Cache-Control", "public, max-age=31536000, immutable")
|
||||
.setHeader("X-SEO-Mode-Preview-Cache", snapshot.status)
|
||||
.type("image/png")
|
||||
.send(Buffer.from(snapshot.body));
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
response.status(503).type("text/plain").send("Preview snapshot failed.");
|
||||
}
|
||||
});
|
||||
|
||||
previewRouter.get(/^\/projects\/([^/]+)\/site\/?(.*)$/i, async (request, response) => {
|
||||
try {
|
||||
const params = request.params as Record<string, string | undefined>;
|
||||
const projectId = params[0] ?? "";
|
||||
const rawAssetPath = params[1] ?? "";
|
||||
const focusSelector = getQueryString(request.query.focus);
|
||||
const focusIndex = getQueryInteger(request.query.focusIndex);
|
||||
const source = await getLatestProjectSource(projectId);
|
||||
|
||||
if (!source) {
|
||||
sendNotFound(response);
|
||||
return;
|
||||
}
|
||||
|
||||
const asset = await readProjectAsset(source, rawAssetPath || "index.html");
|
||||
|
||||
if (!asset) {
|
||||
sendNotFound(response);
|
||||
return;
|
||||
}
|
||||
|
||||
const contentType = asset.contentType ?? getMimeType(asset.path);
|
||||
|
||||
if (isHtml(asset.path, contentType)) {
|
||||
sendAsset(
|
||||
response,
|
||||
asset,
|
||||
rewriteHtml(textFromAsset(asset), projectId, asset.path, source.config, { focusIndex, focusSelector }),
|
||||
"text/html; charset=utf-8"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (isCss(asset.path, contentType)) {
|
||||
sendAsset(response, asset, rewriteCss(textFromAsset(asset), projectId, asset.path, source.config), "text/css; charset=utf-8");
|
||||
return;
|
||||
}
|
||||
|
||||
sendAsset(response, asset, isTextLike(asset.path, contentType) ? textFromAsset(asset) : asset.body, contentType);
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
response.status(500).type("text/plain").send("Preview asset failed.");
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
import crypto from "node:crypto";
|
||||
import { chromium, type Browser } from "playwright-core";
|
||||
import { env } from "../config/env.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
|
||||
type SnapshotRequest = {
|
||||
cacheVersion: string;
|
||||
focusIndex: number | null;
|
||||
focusSelector: string | null;
|
||||
origin: string;
|
||||
projectId: string;
|
||||
sourcePath: string;
|
||||
viewportHeight: number;
|
||||
viewportWidth: number;
|
||||
};
|
||||
|
||||
type QueueTask<T> = {
|
||||
reject: (error: unknown) => void;
|
||||
resolve: (value: T) => void;
|
||||
run: () => Promise<T>;
|
||||
};
|
||||
|
||||
const inFlightSnapshots = new Map<string, Promise<Uint8Array>>();
|
||||
const queue: QueueTask<Uint8Array>[] = [];
|
||||
let activeTasks = 0;
|
||||
let browserPromise: Promise<Browser> | null = null;
|
||||
|
||||
function encodePathname(value: string) {
|
||||
return value
|
||||
.split("/")
|
||||
.filter(Boolean)
|
||||
.map((part) => encodeURIComponent(part))
|
||||
.join("/");
|
||||
}
|
||||
|
||||
function getSnapshotCacheKey(input: SnapshotRequest) {
|
||||
const hash = crypto
|
||||
.createHash("sha256")
|
||||
.update(
|
||||
JSON.stringify({
|
||||
focusIndex: input.focusIndex,
|
||||
focusSelector: input.focusSelector,
|
||||
sourcePath: input.sourcePath,
|
||||
viewportHeight: input.viewportHeight,
|
||||
viewportWidth: input.viewportWidth
|
||||
})
|
||||
)
|
||||
.digest("hex")
|
||||
.slice(0, 32);
|
||||
|
||||
return `preview-cache/${input.projectId}/${input.cacheVersion}/${hash}.png`;
|
||||
}
|
||||
|
||||
function buildPreviewUrl(input: SnapshotRequest) {
|
||||
const url = new URL(`/preview/projects/${encodeURIComponent(input.projectId)}/site/${encodePathname(input.sourcePath)}`, input.origin);
|
||||
|
||||
if (input.focusSelector) {
|
||||
url.searchParams.set("focus", input.focusSelector);
|
||||
}
|
||||
|
||||
if (input.focusIndex != null) {
|
||||
url.searchParams.set("focusIndex", String(input.focusIndex));
|
||||
}
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
async function getBrowser() {
|
||||
if (!browserPromise) {
|
||||
browserPromise = chromium.launch({
|
||||
executablePath: env.preview.chromePath,
|
||||
headless: true,
|
||||
args: ["--disable-dev-shm-usage", "--no-sandbox"]
|
||||
});
|
||||
}
|
||||
|
||||
return browserPromise;
|
||||
}
|
||||
|
||||
function pumpQueue() {
|
||||
while (activeTasks < env.preview.snapshotConcurrency && queue.length > 0) {
|
||||
const task = queue.shift();
|
||||
|
||||
if (!task) {
|
||||
return;
|
||||
}
|
||||
|
||||
activeTasks += 1;
|
||||
void task
|
||||
.run()
|
||||
.then(task.resolve, task.reject)
|
||||
.finally(() => {
|
||||
activeTasks -= 1;
|
||||
pumpQueue();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueSnapshot(run: () => Promise<Uint8Array>) {
|
||||
return new Promise<Uint8Array>((resolve, reject) => {
|
||||
queue.push({ reject, resolve, run });
|
||||
pumpQueue();
|
||||
});
|
||||
}
|
||||
|
||||
async function readCachedSnapshot(key: string) {
|
||||
try {
|
||||
const cached = await storageProvider.getObject(key);
|
||||
|
||||
if (cached.body.byteLength > 0) {
|
||||
return cached.body;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function generateSnapshot(input: SnapshotRequest, cacheKey: string) {
|
||||
const browser = await getBrowser();
|
||||
const page = await browser.newPage({
|
||||
deviceScaleFactor: 1,
|
||||
viewport: {
|
||||
height: input.viewportHeight,
|
||||
width: input.viewportWidth
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
page.setDefaultTimeout(18_000);
|
||||
await page.goto(buildPreviewUrl(input), { timeout: 25_000, waitUntil: "domcontentloaded" });
|
||||
await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => undefined);
|
||||
await page.waitForTimeout(env.preview.snapshotWaitMs);
|
||||
|
||||
const screenshot = await page.screenshot({
|
||||
animations: "disabled",
|
||||
fullPage: false,
|
||||
type: "png"
|
||||
});
|
||||
const body = new Uint8Array(screenshot);
|
||||
|
||||
await storageProvider.putObject({
|
||||
body,
|
||||
contentType: "image/png",
|
||||
key: cacheKey
|
||||
});
|
||||
|
||||
return body;
|
||||
} finally {
|
||||
await page.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getPreviewSnapshot(input: SnapshotRequest) {
|
||||
const cacheKey = getSnapshotCacheKey(input);
|
||||
const cached = await readCachedSnapshot(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
return {
|
||||
body: cached,
|
||||
cacheKey,
|
||||
status: "hit" as const
|
||||
};
|
||||
}
|
||||
|
||||
const existing = inFlightSnapshots.get(cacheKey);
|
||||
|
||||
if (existing) {
|
||||
return {
|
||||
body: await existing,
|
||||
cacheKey,
|
||||
status: "joined" as const
|
||||
};
|
||||
}
|
||||
|
||||
const generated = enqueueSnapshot(() => generateSnapshot(input, cacheKey));
|
||||
inFlightSnapshots.set(cacheKey, generated);
|
||||
|
||||
try {
|
||||
return {
|
||||
body: await generated,
|
||||
cacheKey,
|
||||
status: "miss" as const
|
||||
};
|
||||
} finally {
|
||||
inFlightSnapshots.delete(cacheKey);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import { constants } from "node:fs";
|
||||
import { access, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export class LocalFolderValidationError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "LocalFolderValidationError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function validateLocalFolderPath(inputPath: string) {
|
||||
const trimmedPath = inputPath.trim();
|
||||
|
||||
if (!trimmedPath) {
|
||||
throw new LocalFolderValidationError("Укажи путь к локальной папке проекта.");
|
||||
}
|
||||
|
||||
if (!path.isAbsolute(trimmedPath)) {
|
||||
throw new LocalFolderValidationError("Путь к локальной папке должен быть абсолютным.");
|
||||
}
|
||||
|
||||
const resolvedPath = path.resolve(trimmedPath);
|
||||
|
||||
let stats;
|
||||
try {
|
||||
stats = await stat(resolvedPath);
|
||||
} catch {
|
||||
throw new LocalFolderValidationError("Такой папки не существует или сервер не может её прочитать.");
|
||||
}
|
||||
|
||||
if (!stats.isDirectory()) {
|
||||
throw new LocalFolderValidationError("Указанный путь должен вести именно к папке, а не к файлу.");
|
||||
}
|
||||
|
||||
try {
|
||||
await access(resolvedPath, constants.R_OK);
|
||||
} catch {
|
||||
throw new LocalFolderValidationError("У сервера нет прав на чтение этой папки.");
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
}
|
||||
|
|
@ -0,0 +1,712 @@
|
|||
import type { PoolClient } from "pg";
|
||||
import crypto from "node:crypto";
|
||||
import path from "node:path";
|
||||
import { pool } from "../db/client.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
import { validateLocalFolderPath } from "./localFolder.js";
|
||||
|
||||
type ProjectRow = {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
source_id: string | null;
|
||||
source_type: string | null;
|
||||
source_config: Record<string, unknown> | null;
|
||||
source_created_at: Date | null;
|
||||
source_updated_at: Date | null;
|
||||
};
|
||||
|
||||
type ProjectRecord = {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
type SourceRecord = {
|
||||
id: string;
|
||||
type: string;
|
||||
config: Record<string, unknown>;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
type SourceConfigRow = {
|
||||
config: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type ProjectSourceCopyRow = {
|
||||
id: string;
|
||||
name: string;
|
||||
source_id: string;
|
||||
source_type: string;
|
||||
source_config: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type ProjectSummary = {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
source: {
|
||||
id: string;
|
||||
type: string;
|
||||
path: string | null;
|
||||
config: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
} | null;
|
||||
};
|
||||
|
||||
export type CreateProjectInput = {
|
||||
name?: string;
|
||||
sourcePath: string;
|
||||
};
|
||||
|
||||
export type BrowserFolderFileInput = {
|
||||
relativePath: string;
|
||||
contentBase64: string;
|
||||
contentType?: string;
|
||||
size: number;
|
||||
lastModified?: number;
|
||||
};
|
||||
|
||||
export type CreateBrowserFolderProjectInput = {
|
||||
name?: string;
|
||||
rootName: string;
|
||||
files: BrowserFolderFileInput[];
|
||||
};
|
||||
|
||||
export class ProjectNameConflictError extends Error {
|
||||
constructor(name: string) {
|
||||
super(`Проект с названием «${name}» уже есть.`);
|
||||
this.name = "ProjectNameConflictError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ProjectCopyError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "ProjectCopyError";
|
||||
}
|
||||
}
|
||||
|
||||
const CYRILLIC_TO_LATIN: Record<string, string> = {
|
||||
а: "a",
|
||||
б: "b",
|
||||
в: "v",
|
||||
г: "g",
|
||||
д: "d",
|
||||
е: "e",
|
||||
ё: "e",
|
||||
ж: "zh",
|
||||
з: "z",
|
||||
и: "i",
|
||||
й: "y",
|
||||
к: "k",
|
||||
л: "l",
|
||||
м: "m",
|
||||
н: "n",
|
||||
о: "o",
|
||||
п: "p",
|
||||
р: "r",
|
||||
с: "s",
|
||||
т: "t",
|
||||
у: "u",
|
||||
ф: "f",
|
||||
х: "h",
|
||||
ц: "ts",
|
||||
ч: "ch",
|
||||
ш: "sh",
|
||||
щ: "sch",
|
||||
ъ: "",
|
||||
ы: "y",
|
||||
ь: "",
|
||||
э: "e",
|
||||
ю: "yu",
|
||||
я: "ya"
|
||||
};
|
||||
|
||||
const MAX_PROJECT_NAME_LENGTH = 120;
|
||||
|
||||
function toIsoDate(value: Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
function getSourcePath(config: Record<string, unknown>) {
|
||||
return typeof config.path === "string" ? config.path : null;
|
||||
}
|
||||
|
||||
function sanitizeObjectPath(value: string) {
|
||||
const normalizedPath = value.replace(/\\/g, "/").replace(/^\/+/, "");
|
||||
const segments = normalizedPath.split("/").filter(Boolean);
|
||||
|
||||
if (segments.length === 0) {
|
||||
throw new Error("Файл без относительного пути нельзя импортировать.");
|
||||
}
|
||||
|
||||
if (segments.some((segment) => segment === "." || segment === "..")) {
|
||||
throw new Error("В импорте найден небезопасный путь файла.");
|
||||
}
|
||||
|
||||
return segments.join("/");
|
||||
}
|
||||
|
||||
function mapProjectRow(row: ProjectRow): ProjectSummary {
|
||||
const sourceConfig = row.source_config ?? {};
|
||||
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
slug: row.slug,
|
||||
createdAt: toIsoDate(row.created_at),
|
||||
updatedAt: toIsoDate(row.updated_at),
|
||||
source:
|
||||
row.source_id && row.source_type && row.source_created_at && row.source_updated_at
|
||||
? {
|
||||
id: row.source_id,
|
||||
type: row.source_type,
|
||||
path: getSourcePath(sourceConfig),
|
||||
config: sourceConfig,
|
||||
createdAt: toIsoDate(row.source_created_at),
|
||||
updatedAt: toIsoDate(row.source_updated_at)
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
|
||||
function mapProjectRecord(project: ProjectRecord, source: SourceRecord): ProjectSummary {
|
||||
return {
|
||||
id: project.id,
|
||||
name: project.name,
|
||||
slug: project.slug,
|
||||
createdAt: toIsoDate(project.created_at),
|
||||
updatedAt: toIsoDate(project.updated_at),
|
||||
source: {
|
||||
id: source.id,
|
||||
type: source.type,
|
||||
path: getSourcePath(source.config),
|
||||
config: source.config,
|
||||
createdAt: toIsoDate(source.created_at),
|
||||
updatedAt: toIsoDate(source.updated_at)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function selectProjectById(client: PoolClient, projectId: string) {
|
||||
const result = await client.query<ProjectRow>(
|
||||
`
|
||||
select
|
||||
p.id,
|
||||
p.name,
|
||||
p.slug,
|
||||
p.created_at,
|
||||
p.updated_at,
|
||||
ps.id as source_id,
|
||||
ps.type as source_type,
|
||||
ps.config as source_config,
|
||||
ps.created_at as source_created_at,
|
||||
ps.updated_at as source_updated_at
|
||||
from projects p
|
||||
left join lateral (
|
||||
select id, type, config, created_at, updated_at
|
||||
from project_sources
|
||||
where project_id = p.id
|
||||
order by created_at asc
|
||||
limit 1
|
||||
) ps on true
|
||||
where p.id = $1;
|
||||
`,
|
||||
[projectId]
|
||||
);
|
||||
|
||||
return result.rows[0] ? mapProjectRow(result.rows[0]) : null;
|
||||
}
|
||||
|
||||
async function selectProjectSourceForCopy(client: PoolClient, projectId: string) {
|
||||
const result = await client.query<ProjectSourceCopyRow>(
|
||||
`
|
||||
select
|
||||
p.id,
|
||||
p.name,
|
||||
ps.id as source_id,
|
||||
ps.type as source_type,
|
||||
ps.config as source_config
|
||||
from projects p
|
||||
join lateral (
|
||||
select id, type, config
|
||||
from project_sources
|
||||
where project_id = p.id
|
||||
order by created_at asc
|
||||
limit 1
|
||||
) ps on true
|
||||
where p.id = $1;
|
||||
`,
|
||||
[projectId]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function insertProjectWithSource(
|
||||
client: PoolClient,
|
||||
input: {
|
||||
name: string;
|
||||
sourceType: "local_folder" | "browser_folder";
|
||||
sourceConfig: Record<string, unknown>;
|
||||
}
|
||||
) {
|
||||
const slug = await buildUniqueSlug(client, input.name);
|
||||
const projectResult = await client.query<ProjectRecord>(
|
||||
`
|
||||
insert into projects (name, slug)
|
||||
values ($1, $2)
|
||||
returning id, name, slug, created_at, updated_at;
|
||||
`,
|
||||
[input.name, slug]
|
||||
);
|
||||
const project = projectResult.rows[0];
|
||||
|
||||
if (!project) {
|
||||
throw new Error("Не удалось создать проект.");
|
||||
}
|
||||
|
||||
const sourceResult = await client.query<SourceRecord>(
|
||||
`
|
||||
insert into project_sources (project_id, type, config)
|
||||
values ($1, $2, $3)
|
||||
returning id, type, config, created_at, updated_at;
|
||||
`,
|
||||
[project.id, input.sourceType, input.sourceConfig]
|
||||
);
|
||||
const source = sourceResult.rows[0];
|
||||
|
||||
if (!source) {
|
||||
throw new Error("Не удалось сохранить источник проекта.");
|
||||
}
|
||||
|
||||
return { project, source };
|
||||
}
|
||||
|
||||
function transliterate(value: string) {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.split("")
|
||||
.map((char) => CYRILLIC_TO_LATIN[char] ?? char)
|
||||
.join("");
|
||||
}
|
||||
|
||||
function slugifyName(name: string) {
|
||||
const slug = transliterate(name)
|
||||
.normalize("NFKD")
|
||||
.replace(/[\u0300-\u036f]/g, "")
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 64);
|
||||
|
||||
return slug || "project";
|
||||
}
|
||||
|
||||
async function slugExists(client: PoolClient, slug: string, excludeProjectId?: string) {
|
||||
const result = excludeProjectId
|
||||
? await client.query<{ exists: boolean }>(
|
||||
"select exists(select 1 from projects where slug = $1 and id <> $2) as exists",
|
||||
[slug, excludeProjectId]
|
||||
)
|
||||
: await client.query<{ exists: boolean }>(
|
||||
"select exists(select 1 from projects where slug = $1) as exists",
|
||||
[slug]
|
||||
);
|
||||
|
||||
return result.rows[0]?.exists ?? false;
|
||||
}
|
||||
|
||||
async function projectNameExists(client: PoolClient, name: string, excludeProjectId?: string) {
|
||||
const result = excludeProjectId
|
||||
? await client.query<{ exists: boolean }>(
|
||||
"select exists(select 1 from projects where lower(name) = lower($1) and id <> $2) as exists",
|
||||
[name, excludeProjectId]
|
||||
)
|
||||
: await client.query<{ exists: boolean }>(
|
||||
"select exists(select 1 from projects where lower(name) = lower($1)) as exists",
|
||||
[name]
|
||||
);
|
||||
|
||||
return result.rows[0]?.exists ?? false;
|
||||
}
|
||||
|
||||
async function assertProjectNameAvailable(client: PoolClient, name: string, excludeProjectId?: string) {
|
||||
if (await projectNameExists(client, name, excludeProjectId)) {
|
||||
throw new ProjectNameConflictError(name);
|
||||
}
|
||||
}
|
||||
|
||||
async function buildUniqueSlug(client: PoolClient, name: string, excludeProjectId?: string) {
|
||||
const baseSlug = slugifyName(name);
|
||||
let suffix = 1;
|
||||
|
||||
while (suffix < 1000) {
|
||||
const suffixText = suffix === 1 ? "" : `-${suffix}`;
|
||||
const candidate = `${baseSlug.slice(0, 64 - suffixText.length)}${suffixText}`;
|
||||
|
||||
if (!(await slugExists(client, candidate, excludeProjectId))) {
|
||||
return candidate;
|
||||
}
|
||||
|
||||
suffix += 1;
|
||||
}
|
||||
|
||||
return `${baseSlug.slice(0, 50)}-${Date.now()}`;
|
||||
}
|
||||
|
||||
function getCopyBaseName(originalName: string) {
|
||||
let baseName = originalName.trim() || "project";
|
||||
let previousBaseName = "";
|
||||
|
||||
while (baseName && baseName !== previousBaseName) {
|
||||
previousBaseName = baseName;
|
||||
baseName = baseName
|
||||
.replace(/^(?:(?:\d{2}|[1-9]\d{0,2})\s*-\s*копия\s*-\s*|Копия(?:\s+\d+)?\s*-\s*|[1-9]\d{0,2}\s*-\s*)/iu, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
return baseName || "project";
|
||||
}
|
||||
|
||||
function buildCopyNameCandidate(originalName: string, copyIndex: number) {
|
||||
const prefix = `${String(copyIndex).padStart(2, "0")} - копия - `;
|
||||
const normalizedOriginalName = getCopyBaseName(originalName);
|
||||
const maxOriginalNameLength = Math.max(1, MAX_PROJECT_NAME_LENGTH - prefix.length);
|
||||
|
||||
return `${prefix}${normalizedOriginalName.slice(0, maxOriginalNameLength)}`;
|
||||
}
|
||||
|
||||
async function buildUniqueCopyName(client: PoolClient, originalName: string) {
|
||||
let copyIndex = 1;
|
||||
|
||||
while (copyIndex < 1000) {
|
||||
const candidate = buildCopyNameCandidate(originalName, copyIndex);
|
||||
|
||||
if (!(await projectNameExists(client, candidate))) {
|
||||
return candidate;
|
||||
}
|
||||
|
||||
copyIndex += 1;
|
||||
}
|
||||
|
||||
return buildCopyNameCandidate(`${crypto.randomUUID().slice(0, 8)}-${originalName}`, 1);
|
||||
}
|
||||
|
||||
export async function listProjects() {
|
||||
const result = await pool.query<ProjectRow>(`
|
||||
select
|
||||
p.id,
|
||||
p.name,
|
||||
p.slug,
|
||||
p.created_at,
|
||||
p.updated_at,
|
||||
ps.id as source_id,
|
||||
ps.type as source_type,
|
||||
ps.config as source_config,
|
||||
ps.created_at as source_created_at,
|
||||
ps.updated_at as source_updated_at
|
||||
from projects p
|
||||
left join lateral (
|
||||
select id, type, config, created_at, updated_at
|
||||
from project_sources
|
||||
where project_id = p.id
|
||||
order by created_at asc
|
||||
limit 1
|
||||
) ps on true
|
||||
order by p.created_at desc;
|
||||
`);
|
||||
|
||||
return result.rows.map(mapProjectRow);
|
||||
}
|
||||
|
||||
export async function createProject(input: CreateProjectInput) {
|
||||
const resolvedSourcePath = await validateLocalFolderPath(input.sourcePath);
|
||||
const name = input.name?.trim() || path.basename(resolvedSourcePath) || "project";
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
await client.query("begin");
|
||||
await assertProjectNameAvailable(client, name);
|
||||
const { project, source } = await insertProjectWithSource(client, {
|
||||
name,
|
||||
sourceType: "local_folder",
|
||||
sourceConfig: {
|
||||
path: resolvedSourcePath,
|
||||
originalPath: input.sourcePath.trim()
|
||||
}
|
||||
});
|
||||
|
||||
await client.query("commit");
|
||||
return mapProjectRecord(project, source);
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function createBrowserFolderProject(input: CreateBrowserFolderProjectInput) {
|
||||
const rootName = input.rootName.trim() || "browser-folder";
|
||||
const name = input.name?.trim() || rootName;
|
||||
const files = input.files.map((file) => ({
|
||||
...file,
|
||||
relativePath: sanitizeObjectPath(file.relativePath)
|
||||
}));
|
||||
const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
|
||||
const client = await pool.connect();
|
||||
|
||||
if (files.length === 0) {
|
||||
throw new Error("В выбранной папке нет файлов для импорта.");
|
||||
}
|
||||
|
||||
try {
|
||||
await client.query("begin");
|
||||
await assertProjectNameAvailable(client, name);
|
||||
|
||||
const importPrefixSeed = crypto.randomUUID();
|
||||
const importPrefix = `browser-folder-imports/${importPrefixSeed}`;
|
||||
const { project, source } = await insertProjectWithSource(client, {
|
||||
name,
|
||||
sourceType: "browser_folder",
|
||||
sourceConfig: {
|
||||
rootName,
|
||||
importPrefix,
|
||||
fileCount: files.length,
|
||||
totalBytes,
|
||||
importedAt: new Date().toISOString(),
|
||||
sampleFiles: files.slice(0, 20).map((file) => file.relativePath)
|
||||
}
|
||||
});
|
||||
|
||||
await storageProvider.ensureReady();
|
||||
|
||||
for (const file of files) {
|
||||
await storageProvider.putObject({
|
||||
key: `${importPrefix}/${file.relativePath}`,
|
||||
body: Buffer.from(file.contentBase64, "base64"),
|
||||
contentType: file.contentType || "application/octet-stream"
|
||||
});
|
||||
}
|
||||
|
||||
await storageProvider.putObject({
|
||||
key: `${importPrefix}/manifest.json`,
|
||||
body: JSON.stringify(
|
||||
{
|
||||
projectId: project.id,
|
||||
sourceId: source.id,
|
||||
rootName,
|
||||
fileCount: files.length,
|
||||
totalBytes,
|
||||
files: files.map((file) => ({
|
||||
relativePath: file.relativePath,
|
||||
contentType: file.contentType,
|
||||
size: file.size,
|
||||
lastModified: file.lastModified
|
||||
}))
|
||||
},
|
||||
null,
|
||||
2
|
||||
),
|
||||
contentType: "application/json"
|
||||
});
|
||||
|
||||
await client.query("commit");
|
||||
return mapProjectRecord(project, source);
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateProjectName(projectId: string, name: string) {
|
||||
const nextName = name.trim();
|
||||
const client = await pool.connect();
|
||||
|
||||
if (!nextName) {
|
||||
throw new Error("Название проекта обязательно.");
|
||||
}
|
||||
|
||||
try {
|
||||
await client.query("begin");
|
||||
await assertProjectNameAvailable(client, nextName, projectId);
|
||||
|
||||
const nextSlug = await buildUniqueSlug(client, nextName, projectId);
|
||||
const updateResult = await client.query(
|
||||
`
|
||||
update projects
|
||||
set name = $1, slug = $2, updated_at = now()
|
||||
where id = $3
|
||||
returning id;
|
||||
`,
|
||||
[nextName, nextSlug, projectId]
|
||||
);
|
||||
|
||||
if (updateResult.rowCount === 0) {
|
||||
await client.query("rollback");
|
||||
return null;
|
||||
}
|
||||
|
||||
const project = await selectProjectById(client, projectId);
|
||||
|
||||
await client.query("commit");
|
||||
return project;
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function copyProject(projectId: string) {
|
||||
const client = await pool.connect();
|
||||
let copiedImportPrefix: string | null = null;
|
||||
|
||||
try {
|
||||
await client.query("begin");
|
||||
|
||||
const sourceProject = await selectProjectSourceForCopy(client, projectId);
|
||||
|
||||
if (!sourceProject) {
|
||||
await client.query("rollback");
|
||||
return null;
|
||||
}
|
||||
|
||||
const copiedAt = new Date().toISOString();
|
||||
const sourceConfig: Record<string, unknown> = {
|
||||
...sourceProject.source_config,
|
||||
copiedFromProjectId: sourceProject.id,
|
||||
copiedFromSourceId: sourceProject.source_id,
|
||||
copiedAt
|
||||
};
|
||||
let sourceType: "local_folder" | "browser_folder";
|
||||
let sourceImportPrefix: string | null = null;
|
||||
|
||||
if (sourceProject.source_type === "local_folder") {
|
||||
sourceType = "local_folder";
|
||||
} else if (sourceProject.source_type === "browser_folder") {
|
||||
const importPrefix = sourceProject.source_config.importPrefix;
|
||||
|
||||
if (typeof importPrefix !== "string" || importPrefix.length === 0) {
|
||||
throw new ProjectCopyError("У исходного проекта нет корректного prefix импорта.");
|
||||
}
|
||||
|
||||
sourceType = "browser_folder";
|
||||
sourceImportPrefix = importPrefix;
|
||||
copiedImportPrefix = `browser-folder-imports/${crypto.randomUUID()}`;
|
||||
sourceConfig.importPrefix = copiedImportPrefix;
|
||||
sourceConfig.copiedFromImportPrefix = sourceImportPrefix;
|
||||
} else {
|
||||
throw new ProjectCopyError("Копирование этого типа источника пока не поддерживается.");
|
||||
}
|
||||
|
||||
const copyName = await buildUniqueCopyName(client, sourceProject.name);
|
||||
const { project, source } = await insertProjectWithSource(client, {
|
||||
name: copyName,
|
||||
sourceType,
|
||||
sourceConfig
|
||||
});
|
||||
|
||||
if (sourceType === "browser_folder" && sourceImportPrefix && copiedImportPrefix) {
|
||||
const copiedObjects = await storageProvider.copyPrefix(sourceImportPrefix, copiedImportPrefix);
|
||||
|
||||
if (copiedObjects === 0) {
|
||||
throw new ProjectCopyError("В хранилище не найдены файлы исходного проекта.");
|
||||
}
|
||||
|
||||
await storageProvider.putObject({
|
||||
key: `${copiedImportPrefix}/manifest.json`,
|
||||
body: JSON.stringify(
|
||||
{
|
||||
projectId: project.id,
|
||||
sourceId: source.id,
|
||||
copiedFromProjectId: sourceProject.id,
|
||||
copiedFromSourceId: sourceProject.source_id,
|
||||
copiedFromImportPrefix: sourceImportPrefix,
|
||||
copiedAt,
|
||||
rootName: sourceConfig.rootName,
|
||||
fileCount: sourceConfig.fileCount,
|
||||
totalBytes: sourceConfig.totalBytes,
|
||||
sampleFiles: sourceConfig.sampleFiles
|
||||
},
|
||||
null,
|
||||
2
|
||||
),
|
||||
contentType: "application/json"
|
||||
});
|
||||
}
|
||||
|
||||
await client.query("commit");
|
||||
return mapProjectRecord(project, source);
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
|
||||
if (copiedImportPrefix) {
|
||||
try {
|
||||
await storageProvider.deletePrefix(copiedImportPrefix);
|
||||
} catch (cleanupError) {
|
||||
console.warn(`Не удалось подчистить файлы неудачной копии по prefix ${copiedImportPrefix}`, cleanupError);
|
||||
}
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteProject(projectId: string) {
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
const sourceConfigs = await client.query<SourceConfigRow>(
|
||||
`
|
||||
select config
|
||||
from project_sources
|
||||
where project_id = $1 and type = 'browser_folder';
|
||||
`,
|
||||
[projectId]
|
||||
);
|
||||
const importPrefixes = sourceConfigs.rows
|
||||
.map((row) => row.config.importPrefix)
|
||||
.filter((value): value is string => typeof value === "string" && value.length > 0);
|
||||
|
||||
await client.query("begin");
|
||||
|
||||
const deleteResult = await client.query("delete from projects where id = $1", [projectId]);
|
||||
|
||||
if (deleteResult.rowCount === 0) {
|
||||
await client.query("rollback");
|
||||
return false;
|
||||
}
|
||||
|
||||
await client.query("commit");
|
||||
|
||||
for (const prefix of importPrefixes) {
|
||||
try {
|
||||
await storageProvider.deletePrefix(prefix);
|
||||
} catch (error) {
|
||||
console.warn(`Не удалось удалить файлы проекта из хранилища по prefix ${prefix}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,599 @@
|
|||
import { Router } from "express";
|
||||
import type { Response } from "express";
|
||||
import crypto from "node:crypto";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
getLatestProjectScan,
|
||||
ProjectScanError,
|
||||
scanProject,
|
||||
updateProjectPageSelection
|
||||
} from "../scans/projectScanner.js";
|
||||
import { LocalFolderValidationError } from "./localFolder.js";
|
||||
import {
|
||||
copyProject,
|
||||
createBrowserFolderProject,
|
||||
createProject,
|
||||
deleteProject,
|
||||
listProjects,
|
||||
ProjectCopyError,
|
||||
ProjectNameConflictError,
|
||||
updateProjectName
|
||||
} from "./projectRepository.js";
|
||||
import {
|
||||
getOrCreatePageWorkspace,
|
||||
PageWorkspaceError,
|
||||
resetPageWorkspace,
|
||||
savePageWorkspace
|
||||
} from "../workspace/pageWorkspace.js";
|
||||
import {
|
||||
getLatestSemanticAnalysis,
|
||||
getSemanticAnalysisExport,
|
||||
runSemanticAnalysis,
|
||||
SemanticAnalysisError
|
||||
} from "../analysis/semanticAnalysis.js";
|
||||
import { getMarketEnrichmentContract, runMarketEnrichment } from "../market/marketEnrichment.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
|
||||
export const projectsRouter = Router();
|
||||
|
||||
const createProjectSchema = z.object({
|
||||
name: z.string().trim().max(120, "Название слишком длинное.").optional().default(""),
|
||||
sourcePath: z.string().trim().min(1, "Путь к локальной папке обязателен.")
|
||||
});
|
||||
|
||||
const browserFolderFileSchema = z.object({
|
||||
relativePath: z.string().trim().min(1),
|
||||
contentBase64: z.string().min(1),
|
||||
contentType: z.string().optional(),
|
||||
size: z.number().int().nonnegative(),
|
||||
lastModified: z.number().optional()
|
||||
});
|
||||
|
||||
const importBrowserFolderSchema = z.object({
|
||||
name: z.string().trim().max(120, "Название слишком длинное.").optional().default(""),
|
||||
rootName: z.string().trim().min(1, "Не удалось определить имя папки."),
|
||||
files: z.array(browserFolderFileSchema).min(1, "В выбранной папке нет файлов для импорта.")
|
||||
});
|
||||
|
||||
const projectIdSchema = z.string().uuid("Некорректный id проекта.");
|
||||
|
||||
const updateProjectNameSchema = z.object({
|
||||
name: z.string().trim().min(1, "Название проекта обязательно.").max(120, "Название слишком длинное.")
|
||||
});
|
||||
|
||||
const updatePageSelectionSchema = z.object({
|
||||
pageIds: z.array(z.string().uuid("Некорректный id страницы.")).min(1, "Нужно выбрать хотя бы одну страницу."),
|
||||
selected: z.boolean()
|
||||
});
|
||||
|
||||
const updatePageWorkspaceSchema = z.object({
|
||||
workingText: z.string().max(1_000_000, "Рабочий текст слишком большой для текущего MVP.")
|
||||
});
|
||||
|
||||
const semanticExportFormatSchema = z.enum(["csv", "json", "markdown"]).default("json");
|
||||
const scopeLayoutKeySchema = z.string().trim().min(1).max(240);
|
||||
const scopeLayoutSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
viewport: z
|
||||
.object({
|
||||
x: z.number().finite(),
|
||||
y: z.number().finite(),
|
||||
zoom: z.number().finite().positive()
|
||||
})
|
||||
.optional(),
|
||||
nodes: z.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
position: z.object({
|
||||
x: z.number().finite(),
|
||||
y: z.number().finite()
|
||||
}),
|
||||
width: z.number().finite().positive().optional(),
|
||||
height: z.number().finite().positive().optional()
|
||||
})
|
||||
)
|
||||
});
|
||||
|
||||
const MAX_BROWSER_FOLDER_FILES = 1500;
|
||||
const MAX_BROWSER_FOLDER_BYTES = 100 * 1024 * 1024;
|
||||
|
||||
function sendError(response: Response, status: number, message: string) {
|
||||
response.status(status).json({
|
||||
error: {
|
||||
message
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getScopeLayoutObjectKey(projectId: string, layoutKey: string) {
|
||||
const digest = crypto.createHash("sha256").update(layoutKey).digest("hex").slice(0, 40);
|
||||
|
||||
return `project-scope-layouts/${projectId}/${digest}.json`;
|
||||
}
|
||||
|
||||
projectsRouter.get("/", async (_request, response) => {
|
||||
try {
|
||||
response.json({
|
||||
projects: await listProjects()
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить проекты.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/", async (request, response) => {
|
||||
try {
|
||||
const input = createProjectSchema.parse(request.body);
|
||||
const project = await createProject(input);
|
||||
|
||||
response.status(201).json({
|
||||
project
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь данные проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof LocalFolderValidationError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectNameConflictError) {
|
||||
sendError(response, 409, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось создать проект.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/import-folder", async (request, response) => {
|
||||
try {
|
||||
const input = importBrowserFolderSchema.parse(request.body);
|
||||
const totalBytes = input.files.reduce((sum, file) => sum + file.size, 0);
|
||||
|
||||
if (input.files.length > MAX_BROWSER_FOLDER_FILES) {
|
||||
sendError(response, 400, `Слишком много файлов: максимум ${MAX_BROWSER_FOLDER_FILES}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (totalBytes > MAX_BROWSER_FOLDER_BYTES) {
|
||||
sendError(response, 400, "Папка слишком большая для первого импорта: максимум 100 МБ.");
|
||||
return;
|
||||
}
|
||||
|
||||
const project = await createBrowserFolderProject(input);
|
||||
|
||||
response.status(201).json({
|
||||
project
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь данные импорта папки.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectNameConflictError) {
|
||||
sendError(response, 409, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectCopyError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось импортировать папку.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/copy", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const project = await copyProject(projectId);
|
||||
|
||||
if (!project) {
|
||||
sendError(response, 404, "Проект не найден.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(201).json({
|
||||
project
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectNameConflictError) {
|
||||
sendError(response, 409, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof Error && error.message) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось скопировать проект.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.get("/:projectId/scans/latest", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
|
||||
response.json({
|
||||
scan: await getLatestProjectScan(projectId)
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить последний скан проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/scans", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const scan = await scanProject(projectId);
|
||||
|
||||
if (!scan) {
|
||||
sendError(response, 404, "Проект не найден.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(201).json({
|
||||
scan
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectScanError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось просканировать проект.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.get("/:projectId/analysis/semantic/latest", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
|
||||
response.json({
|
||||
analysis: await getLatestSemanticAnalysis(projectId)
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить semantic analysis проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/analysis/semantic", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const analysis = await runSemanticAnalysis(projectId);
|
||||
|
||||
if (!analysis) {
|
||||
sendError(response, 404, "Сначала нужен успешный скан проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(201).json({
|
||||
analysis
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof SemanticAnalysisError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось выполнить semantic analysis проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.get("/:projectId/analysis/semantic/export", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const format = semanticExportFormatSchema.parse(request.query.format);
|
||||
const exportedAnalysis = await getSemanticAnalysisExport(projectId, format);
|
||||
|
||||
if (!exportedAnalysis) {
|
||||
sendError(response, 404, "Сначала нужен semantic analysis проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response
|
||||
.status(200)
|
||||
.setHeader("Content-Type", exportedAnalysis.contentType)
|
||||
.setHeader("Content-Disposition", `attachment; filename="${exportedAnalysis.filename}"`)
|
||||
.send(exportedAnalysis.body);
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный формат экспорта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось выгрузить semantic export проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.get("/:projectId/market-enrichment/latest", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
|
||||
response.json({
|
||||
marketEnrichment: await getMarketEnrichmentContract(projectId)
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить market enrichment проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/market-enrichment", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const marketEnrichment = await runMarketEnrichment(projectId);
|
||||
|
||||
if (!marketEnrichment.semanticRunId) {
|
||||
sendError(response, 404, "Сначала нужен semantic analysis проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(202).json({
|
||||
marketEnrichment
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось запустить market enrichment проекта.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.get("/:projectId/scope-layouts/:layoutKey", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const layoutKey = scopeLayoutKeySchema.parse(request.params.layoutKey);
|
||||
|
||||
try {
|
||||
const rawLayout = await storageProvider.getObjectText(getScopeLayoutObjectKey(projectId, layoutKey));
|
||||
const layout = scopeLayoutSchema.parse(JSON.parse(rawLayout));
|
||||
|
||||
response.json({ layout });
|
||||
} catch {
|
||||
response.json({ layout: null });
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный ключ layout.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить layout workflow.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.put("/:projectId/scope-layouts/:layoutKey", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const layoutKey = scopeLayoutKeySchema.parse(request.params.layoutKey);
|
||||
const layout = scopeLayoutSchema.parse(request.body.layout);
|
||||
|
||||
await storageProvider.putObject({
|
||||
body: JSON.stringify(layout),
|
||||
contentType: "application/json; charset=utf-8",
|
||||
key: getScopeLayoutObjectKey(projectId, layoutKey)
|
||||
});
|
||||
|
||||
response.json({ layout });
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь layout workflow.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось сохранить layout workflow.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.patch("/:projectId/pages/selection", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const input = updatePageSelectionSchema.parse(request.body);
|
||||
const result = await updateProjectPageSelection(projectId, input.pageIds, input.selected);
|
||||
|
||||
response.json(result);
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь выбор страниц.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось обновить выбор страниц.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/pages/:pageId/workspace", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const pageId = projectIdSchema.parse(request.params.pageId);
|
||||
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
|
||||
|
||||
if (!workspace) {
|
||||
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(201).json({
|
||||
workspace
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id страницы.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof PageWorkspaceError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось открыть рабочую зону страницы.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.patch("/:projectId/pages/:pageId/workspace", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const pageId = projectIdSchema.parse(request.params.pageId);
|
||||
const input = updatePageWorkspaceSchema.parse(request.body);
|
||||
const workspace = await savePageWorkspace(projectId, pageId, input.workingText);
|
||||
|
||||
if (!workspace) {
|
||||
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.json({
|
||||
workspace
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь рабочий текст страницы.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof PageWorkspaceError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось сохранить рабочий draft страницы.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.post("/:projectId/pages/:pageId/workspace/reset", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const pageId = projectIdSchema.parse(request.params.pageId);
|
||||
const workspace = await resetPageWorkspace(projectId, pageId);
|
||||
|
||||
if (!workspace) {
|
||||
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.json({
|
||||
workspace
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id страницы.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof PageWorkspaceError) {
|
||||
sendError(response, 400, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось сбросить рабочий draft страницы.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.patch("/:projectId", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const input = updateProjectNameSchema.parse(request.body);
|
||||
const project = await updateProjectName(projectId, input.name);
|
||||
|
||||
if (!project) {
|
||||
sendError(response, 404, "Проект не найден.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.json({
|
||||
project
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь название проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (error instanceof ProjectNameConflictError) {
|
||||
sendError(response, 409, error.message);
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось переименовать проект.");
|
||||
}
|
||||
});
|
||||
|
||||
projectsRouter.delete("/:projectId", async (request, response) => {
|
||||
try {
|
||||
const projectId = projectIdSchema.parse(request.params.projectId);
|
||||
const deleted = await deleteProject(projectId);
|
||||
|
||||
if (!deleted) {
|
||||
sendError(response, 404, "Проект не найден.");
|
||||
return;
|
||||
}
|
||||
|
||||
response.status(204).send();
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось удалить проект.");
|
||||
}
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,63 @@
|
|||
import { readdir, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pool } from "../db/client.js";
|
||||
|
||||
const currentFile = fileURLToPath(import.meta.url);
|
||||
const migrationsDir = path.resolve(path.dirname(currentFile), "../../migrations");
|
||||
|
||||
async function ensureMigrationsTable() {
|
||||
await pool.query(`
|
||||
create table if not exists schema_migrations (
|
||||
id text primary key,
|
||||
applied_at timestamptz not null default now()
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
async function hasMigration(id: string) {
|
||||
const result = await pool.query<{ exists: boolean }>(
|
||||
"select exists(select 1 from schema_migrations where id = $1) as exists",
|
||||
[id]
|
||||
);
|
||||
return result.rows[0]?.exists ?? false;
|
||||
}
|
||||
|
||||
async function applyMigration(id: string, sql: string) {
|
||||
await pool.query("begin");
|
||||
try {
|
||||
await pool.query(sql);
|
||||
await pool.query("insert into schema_migrations (id) values ($1)", [id]);
|
||||
await pool.query("commit");
|
||||
} catch (error) {
|
||||
await pool.query("rollback");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await ensureMigrationsTable();
|
||||
|
||||
const files = (await readdir(migrationsDir))
|
||||
.filter((file) => file.endsWith(".sql"))
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
|
||||
for (const file of files) {
|
||||
if (await hasMigration(file)) {
|
||||
console.log(`Skipping ${file}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const sql = await readFile(path.join(migrationsDir, file), "utf8");
|
||||
await applyMigration(file, sql);
|
||||
console.log(`Applied ${file}`);
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch(async (error) => {
|
||||
console.error(error);
|
||||
await pool.end();
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
@ -0,0 +1,764 @@
|
|||
import { env } from "../config/env.js";
|
||||
import { pool } from "../db/client.js";
|
||||
|
||||
export type ExternalServiceId =
|
||||
| "yandex_cloud"
|
||||
| "yandex_metrica"
|
||||
| "yandex_search_serp"
|
||||
| "yandex_webmaster"
|
||||
| "yandex_wordstat";
|
||||
|
||||
type ExternalServiceStatus = "configured" | "disabled" | "missing_required" | "not_implemented";
|
||||
type ExternalServiceFieldType = "number" | "password" | "text" | "url";
|
||||
|
||||
type ExternalServiceField = {
|
||||
id: string;
|
||||
label: string;
|
||||
type: ExternalServiceFieldType;
|
||||
required: boolean;
|
||||
placeholder: string;
|
||||
help: string;
|
||||
};
|
||||
|
||||
type ExternalServiceDefinition = {
|
||||
id: ExternalServiceId;
|
||||
label: string;
|
||||
description: string;
|
||||
implementationStatus: "active" | "planned";
|
||||
defaultMode: string;
|
||||
modes: Array<{
|
||||
id: string;
|
||||
label: string;
|
||||
}>;
|
||||
docs: Array<{
|
||||
label: string;
|
||||
url: string;
|
||||
}>;
|
||||
publicFields: ExternalServiceField[];
|
||||
secretFields: ExternalServiceField[];
|
||||
};
|
||||
|
||||
type ExternalServiceSettingsRow = {
|
||||
service_id: ExternalServiceId;
|
||||
enabled: boolean;
|
||||
mode: string;
|
||||
public_config: Record<string, unknown>;
|
||||
secret_config: Record<string, unknown>;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
export type ExternalServiceSettings = {
|
||||
schemaVersion: "external-service-settings.v1";
|
||||
services: Array<{
|
||||
id: ExternalServiceId;
|
||||
label: string;
|
||||
description: string;
|
||||
implementationStatus: ExternalServiceDefinition["implementationStatus"];
|
||||
status: ExternalServiceStatus;
|
||||
enabled: boolean;
|
||||
mode: string;
|
||||
modes: ExternalServiceDefinition["modes"];
|
||||
dependsOn: ExternalServiceId[];
|
||||
docs: ExternalServiceDefinition["docs"];
|
||||
publicFields: ExternalServiceField[];
|
||||
secretFields: ExternalServiceField[];
|
||||
publicConfig: Record<string, string>;
|
||||
secrets: Record<
|
||||
string,
|
||||
{
|
||||
configured: boolean;
|
||||
}
|
||||
>;
|
||||
missingFields: string[];
|
||||
updatedAt: string | null;
|
||||
}>;
|
||||
};
|
||||
|
||||
export type UpdateExternalServiceSettingsInput = {
|
||||
enabled: boolean;
|
||||
mode: string;
|
||||
publicConfig?: Record<string, unknown>;
|
||||
secrets?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export type WordstatRuntimeConfig =
|
||||
| {
|
||||
provider: "disabled";
|
||||
mode: "disabled";
|
||||
}
|
||||
| {
|
||||
provider: "mcp_kv";
|
||||
mode: "mcp_kv";
|
||||
mcpEndpoint: string;
|
||||
}
|
||||
| {
|
||||
provider: "yandex_api";
|
||||
mode: "yandex_search_api";
|
||||
apiBaseUrl: string;
|
||||
apiToken: string;
|
||||
authType: "api_key" | "iam_token";
|
||||
folderId: string;
|
||||
regionIds: string[];
|
||||
devices: string[];
|
||||
numPhrases: number;
|
||||
};
|
||||
|
||||
const WORDSTAT_BASE_URL = "https://searchapi.api.cloud.yandex.net/v2/wordstat";
|
||||
const WEB_SEARCH_BASE_URL = "https://searchapi.api.cloud.yandex.net/v2/web/search";
|
||||
const WEBMASTER_BASE_URL = "https://api.webmaster.yandex.net/v4";
|
||||
const METRICA_BASE_URL = "https://api-metrika.yandex.net";
|
||||
|
||||
const serviceDefinitions: Record<ExternalServiceId, ExternalServiceDefinition> = {
|
||||
yandex_cloud: {
|
||||
id: "yandex_cloud",
|
||||
label: "Yandex Cloud credentials",
|
||||
description: "Один общий credential для Search API слоёв: Wordstat сейчас, SERP позже.",
|
||||
implementationStatus: "active",
|
||||
defaultMode: "api_key",
|
||||
modes: [
|
||||
{ id: "disabled", label: "Отключено" },
|
||||
{ id: "api_key", label: "API key" },
|
||||
{ id: "iam_token", label: "IAM token" }
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
label: "API keys",
|
||||
url: "https://yandex.cloud/en/docs/iam/concepts/authorization/api-key"
|
||||
},
|
||||
{
|
||||
label: "Folder ID",
|
||||
url: "https://yandex.cloud/en/docs/resource-manager/operations/folder/get-id"
|
||||
}
|
||||
],
|
||||
publicFields: [
|
||||
{
|
||||
id: "folderId",
|
||||
label: "Yandex Cloud folderId",
|
||||
type: "text",
|
||||
required: true,
|
||||
placeholder: "b1g...",
|
||||
help: "ID каталога Yandex Cloud. Его используют Wordstat и SERP Search API."
|
||||
}
|
||||
],
|
||||
secretFields: [
|
||||
{
|
||||
id: "apiKey",
|
||||
label: "API key",
|
||||
type: "password",
|
||||
required: false,
|
||||
placeholder: "Api-Key ...",
|
||||
help: "Yandex Cloud API key сервисного аккаунта с доступом к Search API."
|
||||
},
|
||||
{
|
||||
id: "iamToken",
|
||||
label: "IAM token",
|
||||
type: "password",
|
||||
required: false,
|
||||
placeholder: "Bearer token",
|
||||
help: "Альтернатива API key. Для prototype можно хранить локально, позже уйдёт в vault."
|
||||
}
|
||||
]
|
||||
},
|
||||
yandex_wordstat: {
|
||||
id: "yandex_wordstat",
|
||||
label: "Yandex Wordstat",
|
||||
description: "Частотность, похожие запросы, динамика и региональная раскладка через Yandex Search API Wordstat.",
|
||||
implementationStatus: "active",
|
||||
defaultMode: "yandex_search_api",
|
||||
modes: [
|
||||
{ id: "disabled", label: "Отключено" },
|
||||
{ id: "yandex_search_api", label: "Yandex Search API Wordstat" },
|
||||
{ id: "mcp_kv", label: "MCP/adapter endpoint" }
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
label: "Wordstat GetTop",
|
||||
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/Wordstat/getTop"
|
||||
},
|
||||
{
|
||||
label: "Regions",
|
||||
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/Wordstat/getRegionsDistribution"
|
||||
},
|
||||
{
|
||||
label: "Dynamics",
|
||||
url: "https://aistudio.yandex.ru/docs/ru/search-api/api-ref/Wordstat/getDynamics"
|
||||
}
|
||||
],
|
||||
publicFields: [
|
||||
{
|
||||
id: "apiBaseUrl",
|
||||
label: "API base URL",
|
||||
type: "url",
|
||||
required: true,
|
||||
placeholder: WORDSTAT_BASE_URL,
|
||||
help: "Root endpoint Wordstat v2. Методы вызываются как /topRequests, /regions, /dynamics."
|
||||
},
|
||||
{
|
||||
id: "regionIds",
|
||||
label: "Region IDs",
|
||||
type: "text",
|
||||
required: false,
|
||||
placeholder: "225",
|
||||
help: "Через запятую. 225 — Россия; можно заменить на нужные регионы после RegionsTree."
|
||||
},
|
||||
{
|
||||
id: "devices",
|
||||
label: "Devices",
|
||||
type: "text",
|
||||
required: false,
|
||||
placeholder: "DEVICE_ALL",
|
||||
help: "Через запятую. Для первого слоя держим DEVICE_ALL."
|
||||
},
|
||||
{
|
||||
id: "numPhrases",
|
||||
label: "Phrases per seed",
|
||||
type: "number",
|
||||
required: false,
|
||||
placeholder: "50",
|
||||
help: "Сколько Wordstat rows запрашивать для одной seed-фразы."
|
||||
},
|
||||
{
|
||||
id: "mcpEndpoint",
|
||||
label: "MCP endpoint",
|
||||
type: "url",
|
||||
required: false,
|
||||
placeholder: "http://localhost:4120/wordstat/collect",
|
||||
help: "Используется только в режиме MCP/adapter endpoint."
|
||||
}
|
||||
],
|
||||
secretFields: []
|
||||
},
|
||||
yandex_search_serp: {
|
||||
id: "yandex_search_serp",
|
||||
label: "Yandex SERP",
|
||||
description: "Поисковая выдача, конкуренты, сниппеты и типы страниц через Yandex Search API WebSearch.",
|
||||
implementationStatus: "planned",
|
||||
defaultMode: "yandex_search_api",
|
||||
modes: [
|
||||
{ id: "disabled", label: "Отключено" },
|
||||
{ id: "yandex_search_api", label: "Yandex Search API WebSearch" }
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
label: "WebSearch.Search",
|
||||
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/WebSearch/search"
|
||||
}
|
||||
],
|
||||
publicFields: [
|
||||
{
|
||||
id: "apiBaseUrl",
|
||||
label: "API endpoint",
|
||||
type: "url",
|
||||
required: true,
|
||||
placeholder: WEB_SEARCH_BASE_URL,
|
||||
help: "REST endpoint для WebSearch.Search."
|
||||
},
|
||||
{
|
||||
id: "searchType",
|
||||
label: "Search type",
|
||||
type: "text",
|
||||
required: false,
|
||||
placeholder: "SEARCH_TYPE_RU",
|
||||
help: "Для российского рынка держим SEARCH_TYPE_RU."
|
||||
}
|
||||
],
|
||||
secretFields: []
|
||||
},
|
||||
yandex_webmaster: {
|
||||
id: "yandex_webmaster",
|
||||
label: "Yandex Webmaster",
|
||||
description: "Индексация, ошибки обхода, sitemap, canonical и статусы страниц после Apply.",
|
||||
implementationStatus: "planned",
|
||||
defaultMode: "oauth",
|
||||
modes: [
|
||||
{ id: "disabled", label: "Отключено" },
|
||||
{ id: "oauth", label: "OAuth token" }
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
label: "Webmaster API",
|
||||
url: "https://yandex.ru/dev/webmaster/doc/en/"
|
||||
}
|
||||
],
|
||||
publicFields: [
|
||||
{
|
||||
id: "apiBaseUrl",
|
||||
label: "API base URL",
|
||||
type: "url",
|
||||
required: true,
|
||||
placeholder: WEBMASTER_BASE_URL,
|
||||
help: "Базовый URL Webmaster API."
|
||||
},
|
||||
{
|
||||
id: "hostId",
|
||||
label: "Host ID",
|
||||
type: "text",
|
||||
required: false,
|
||||
placeholder: "https:example.com:443",
|
||||
help: "ID сайта в Webmaster, можно заполнить позже после discovery."
|
||||
}
|
||||
],
|
||||
secretFields: [
|
||||
{
|
||||
id: "oauthToken",
|
||||
label: "OAuth token",
|
||||
type: "password",
|
||||
required: true,
|
||||
placeholder: "OAuth ...",
|
||||
help: "Токен доступа Webmaster API."
|
||||
}
|
||||
]
|
||||
},
|
||||
yandex_metrica: {
|
||||
id: "yandex_metrica",
|
||||
label: "Yandex Metrica",
|
||||
description: "Фактические показы, клики, CTR, цели и обратная связь после SEO-правок.",
|
||||
implementationStatus: "planned",
|
||||
defaultMode: "oauth",
|
||||
modes: [
|
||||
{ id: "disabled", label: "Отключено" },
|
||||
{ id: "oauth", label: "OAuth token" }
|
||||
],
|
||||
docs: [
|
||||
{
|
||||
label: "Metrica API",
|
||||
url: "https://yandex.ru/dev/metrika/doc/api2/concept/about.html"
|
||||
}
|
||||
],
|
||||
publicFields: [
|
||||
{
|
||||
id: "apiBaseUrl",
|
||||
label: "API base URL",
|
||||
type: "url",
|
||||
required: true,
|
||||
placeholder: METRICA_BASE_URL,
|
||||
help: "Базовый URL Metrica API."
|
||||
},
|
||||
{
|
||||
id: "counterId",
|
||||
label: "Counter ID",
|
||||
type: "text",
|
||||
required: true,
|
||||
placeholder: "12345678",
|
||||
help: "ID счётчика Метрики."
|
||||
}
|
||||
],
|
||||
secretFields: [
|
||||
{
|
||||
id: "oauthToken",
|
||||
label: "OAuth token",
|
||||
type: "password",
|
||||
required: true,
|
||||
placeholder: "OAuth ...",
|
||||
help: "Токен Metrica API."
|
||||
}
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
function toStringRecord(value: Record<string, unknown> | null | undefined) {
|
||||
const output: Record<string, string> = {};
|
||||
|
||||
for (const [key, item] of Object.entries(value ?? {})) {
|
||||
if (item === null || item === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
output[key] = String(item);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function getFieldDefault(serviceId: ExternalServiceId, fieldId: string) {
|
||||
if (serviceId === "yandex_cloud") {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (serviceId === "yandex_wordstat") {
|
||||
const defaults: Record<string, string> = {
|
||||
apiBaseUrl: WORDSTAT_BASE_URL,
|
||||
devices: "DEVICE_ALL",
|
||||
numPhrases: "50",
|
||||
regionIds: "225"
|
||||
};
|
||||
|
||||
return defaults[fieldId] ?? "";
|
||||
}
|
||||
|
||||
if (serviceId === "yandex_search_serp") {
|
||||
const defaults: Record<string, string> = {
|
||||
apiBaseUrl: WEB_SEARCH_BASE_URL,
|
||||
searchType: "SEARCH_TYPE_RU"
|
||||
};
|
||||
|
||||
return defaults[fieldId] ?? "";
|
||||
}
|
||||
|
||||
if (serviceId === "yandex_webmaster") {
|
||||
return fieldId === "apiBaseUrl" ? WEBMASTER_BASE_URL : "";
|
||||
}
|
||||
|
||||
if (serviceId === "yandex_metrica") {
|
||||
return fieldId === "apiBaseUrl" ? METRICA_BASE_URL : "";
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function getDependsOn(serviceId: ExternalServiceId, mode: string): ExternalServiceId[] {
|
||||
if ((serviceId === "yandex_wordstat" || serviceId === "yandex_search_serp") && mode === "yandex_search_api") {
|
||||
return ["yandex_cloud"];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function getPublicConfig(definition: ExternalServiceDefinition, row: ExternalServiceSettingsRow | null) {
|
||||
const config = toStringRecord(row?.public_config);
|
||||
|
||||
for (const field of definition.publicFields) {
|
||||
if (!config[field.id]) {
|
||||
config[field.id] = getFieldDefault(definition.id, field.id);
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
function getMissingFields(
|
||||
definition: ExternalServiceDefinition,
|
||||
enabled: boolean,
|
||||
mode: string,
|
||||
publicConfig: Record<string, string>,
|
||||
secretConfig: Record<string, unknown>,
|
||||
dependencyStatuses: Map<ExternalServiceId, ExternalServiceStatus> = new Map()
|
||||
) {
|
||||
if (!enabled || mode === "disabled") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const field of definition.publicFields) {
|
||||
if (field.id === "mcpEndpoint" && mode !== "mcp_kv") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.id !== "mcpEndpoint" && mode === "mcp_kv") {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (field.required && !publicConfig[field.id]?.trim()) {
|
||||
missing.push(field.id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const dependencyId of getDependsOn(definition.id, mode)) {
|
||||
if (dependencyStatuses.get(dependencyId) !== "configured") {
|
||||
missing.push(dependencyId);
|
||||
}
|
||||
}
|
||||
|
||||
if (definition.id === "yandex_cloud" && (mode === "api_key" || mode === "iam_token")) {
|
||||
const hasApiKey = typeof secretConfig.apiKey === "string" && secretConfig.apiKey.trim().length > 0;
|
||||
const hasIamToken = typeof secretConfig.iamToken === "string" && secretConfig.iamToken.trim().length > 0;
|
||||
|
||||
if (mode === "api_key" && !hasApiKey) {
|
||||
missing.push("apiKey");
|
||||
}
|
||||
|
||||
if (mode === "iam_token" && !hasIamToken) {
|
||||
missing.push("iamToken");
|
||||
}
|
||||
} else {
|
||||
for (const field of definition.secretFields) {
|
||||
const value = secretConfig[field.id];
|
||||
|
||||
if (field.required && !(typeof value === "string" && value.trim().length > 0)) {
|
||||
missing.push(field.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
function getStatus(
|
||||
definition: ExternalServiceDefinition,
|
||||
enabled: boolean,
|
||||
mode: string,
|
||||
missingFields: string[]
|
||||
): ExternalServiceStatus {
|
||||
if (!enabled || mode === "disabled") {
|
||||
return "disabled";
|
||||
}
|
||||
|
||||
if (definition.implementationStatus === "planned") {
|
||||
return "not_implemented";
|
||||
}
|
||||
|
||||
return missingFields.length === 0 ? "configured" : "missing_required";
|
||||
}
|
||||
|
||||
function mapRow(
|
||||
definition: ExternalServiceDefinition,
|
||||
row: ExternalServiceSettingsRow | null,
|
||||
dependencyStatuses: Map<ExternalServiceId, ExternalServiceStatus> = new Map()
|
||||
) {
|
||||
const enabled = row?.enabled ?? false;
|
||||
const mode = row?.mode ?? "disabled";
|
||||
const publicConfig = getPublicConfig(definition, row);
|
||||
const secretConfig = row?.secret_config ?? {};
|
||||
const secrets = Object.fromEntries(
|
||||
definition.secretFields.map((field) => [
|
||||
field.id,
|
||||
{
|
||||
configured: (() => {
|
||||
const value = secretConfig[field.id];
|
||||
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
})()
|
||||
}
|
||||
])
|
||||
);
|
||||
const missingFields = getMissingFields(definition, enabled, mode, publicConfig, secretConfig, dependencyStatuses);
|
||||
|
||||
return {
|
||||
id: definition.id,
|
||||
label: definition.label,
|
||||
description: definition.description,
|
||||
implementationStatus: definition.implementationStatus,
|
||||
status: getStatus(definition, enabled, mode, missingFields),
|
||||
enabled,
|
||||
mode,
|
||||
modes: definition.modes,
|
||||
dependsOn: getDependsOn(definition.id, mode),
|
||||
docs: definition.docs,
|
||||
publicFields: definition.publicFields,
|
||||
secretFields: definition.secretFields,
|
||||
publicConfig,
|
||||
secrets,
|
||||
missingFields,
|
||||
updatedAt: row?.updated_at.toISOString() ?? null
|
||||
};
|
||||
}
|
||||
|
||||
async function getSettingsRows() {
|
||||
const result = await pool.query<ExternalServiceSettingsRow>(
|
||||
`
|
||||
select service_id, enabled, mode, public_config, secret_config, created_at, updated_at
|
||||
from external_service_settings;
|
||||
`
|
||||
);
|
||||
|
||||
return new Map(result.rows.map((row) => [row.service_id, row]));
|
||||
}
|
||||
|
||||
function getDependencyStatuses(rows: Map<ExternalServiceId, ExternalServiceSettingsRow>) {
|
||||
const cloudDefinition = serviceDefinitions.yandex_cloud;
|
||||
const cloudRow = rows.get("yandex_cloud") ?? null;
|
||||
const cloudPublicConfig = getPublicConfig(cloudDefinition, cloudRow);
|
||||
const cloudSecretConfig = cloudRow?.secret_config ?? {};
|
||||
const cloudEnabled = cloudRow?.enabled ?? false;
|
||||
const cloudMode = cloudRow?.mode ?? "disabled";
|
||||
const cloudMissingFields = getMissingFields(
|
||||
cloudDefinition,
|
||||
cloudEnabled,
|
||||
cloudMode,
|
||||
cloudPublicConfig,
|
||||
cloudSecretConfig
|
||||
);
|
||||
|
||||
return new Map<ExternalServiceId, ExternalServiceStatus>([
|
||||
["yandex_cloud", getStatus(cloudDefinition, cloudEnabled, cloudMode, cloudMissingFields)]
|
||||
]);
|
||||
}
|
||||
|
||||
export async function getExternalServiceSettings(): Promise<ExternalServiceSettings> {
|
||||
const rows = await getSettingsRows();
|
||||
const dependencyStatuses = getDependencyStatuses(rows);
|
||||
|
||||
return {
|
||||
schemaVersion: "external-service-settings.v1",
|
||||
services: (Object.keys(serviceDefinitions) as ExternalServiceId[]).map((serviceId) =>
|
||||
mapRow(serviceDefinitions[serviceId], rows.get(serviceId) ?? null, dependencyStatuses)
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
function cleanPublicConfig(definition: ExternalServiceDefinition, value: Record<string, unknown> | undefined) {
|
||||
const allowedFields = new Set(definition.publicFields.map((field) => field.id));
|
||||
const output: Record<string, string> = {};
|
||||
|
||||
for (const [key, item] of Object.entries(value ?? {})) {
|
||||
if (!allowedFields.has(key) || item === null || item === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
output[key] = String(item).trim();
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function mergeSecrets(
|
||||
definition: ExternalServiceDefinition,
|
||||
currentSecrets: Record<string, unknown>,
|
||||
incomingSecrets: Record<string, unknown> | undefined
|
||||
) {
|
||||
const allowedFields = new Set(definition.secretFields.map((field) => field.id));
|
||||
const output: Record<string, string> = {};
|
||||
|
||||
for (const [key, item] of Object.entries(currentSecrets)) {
|
||||
if (allowedFields.has(key) && typeof item === "string" && item.trim()) {
|
||||
output[key] = item;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, item] of Object.entries(incomingSecrets ?? {})) {
|
||||
if (!allowedFields.has(key)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const text = String(item ?? "").trim();
|
||||
|
||||
if (text) {
|
||||
output[key] = text;
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
export async function updateExternalServiceSettings(
|
||||
serviceId: ExternalServiceId,
|
||||
input: UpdateExternalServiceSettingsInput
|
||||
) {
|
||||
const definition = serviceDefinitions[serviceId];
|
||||
const existing = (
|
||||
await pool.query<ExternalServiceSettingsRow>(
|
||||
`
|
||||
select service_id, enabled, mode, public_config, secret_config, created_at, updated_at
|
||||
from external_service_settings
|
||||
where service_id = $1;
|
||||
`,
|
||||
[serviceId]
|
||||
)
|
||||
).rows[0];
|
||||
const mode = definition.modes.some((item) => item.id === input.mode) ? input.mode : definition.defaultMode;
|
||||
const publicConfig = cleanPublicConfig(definition, input.publicConfig);
|
||||
const secretConfig = input.enabled && mode !== "disabled" ? mergeSecrets(definition, existing?.secret_config ?? {}, input.secrets) : {};
|
||||
const result = await pool.query<ExternalServiceSettingsRow>(
|
||||
`
|
||||
insert into external_service_settings (service_id, enabled, mode, public_config, secret_config, updated_at)
|
||||
values ($1, $2, $3, $4, $5, now())
|
||||
on conflict (service_id) do update
|
||||
set enabled = excluded.enabled,
|
||||
mode = excluded.mode,
|
||||
public_config = excluded.public_config,
|
||||
secret_config = excluded.secret_config,
|
||||
updated_at = now()
|
||||
returning service_id, enabled, mode, public_config, secret_config, created_at, updated_at;
|
||||
`,
|
||||
[serviceId, input.enabled, mode, publicConfig, secretConfig]
|
||||
);
|
||||
const row = result.rows[0];
|
||||
|
||||
if (!row) {
|
||||
throw new Error("Не удалось сохранить настройки внешнего сервиса.");
|
||||
}
|
||||
|
||||
const rows = await getSettingsRows();
|
||||
const dependencyStatuses = getDependencyStatuses(rows);
|
||||
|
||||
return mapRow(definition, row, dependencyStatuses);
|
||||
}
|
||||
|
||||
function splitCsv(value: string | undefined, fallback: string[]) {
|
||||
const values = (value ?? "")
|
||||
.split(",")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
return values.length > 0 ? values : fallback;
|
||||
}
|
||||
|
||||
function getNumber(value: string | undefined, fallback: number) {
|
||||
const parsed = Number(value);
|
||||
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.round(parsed) : fallback;
|
||||
}
|
||||
|
||||
export async function getWordstatRuntimeConfig(): Promise<WordstatRuntimeConfig> {
|
||||
const rows = await getSettingsRows();
|
||||
const row = rows.get("yandex_wordstat");
|
||||
const cloudRow = rows.get("yandex_cloud");
|
||||
const definition = serviceDefinitions.yandex_wordstat;
|
||||
const dependencyStatuses = getDependencyStatuses(rows);
|
||||
const mapped = mapRow(definition, row ?? null, dependencyStatuses);
|
||||
|
||||
if (row?.enabled && row.mode === "mcp_kv") {
|
||||
const endpoint = mapped.publicConfig.mcpEndpoint?.trim();
|
||||
|
||||
if (endpoint) {
|
||||
return {
|
||||
provider: "mcp_kv",
|
||||
mode: "mcp_kv",
|
||||
mcpEndpoint: endpoint
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (row?.enabled && row.mode === "yandex_search_api" && mapped.status === "configured") {
|
||||
const cloudPublicConfig = getPublicConfig(serviceDefinitions.yandex_cloud, cloudRow ?? null);
|
||||
const cloudSecretConfig = cloudRow?.secret_config ?? {};
|
||||
const apiKey = typeof cloudSecretConfig.apiKey === "string" ? cloudSecretConfig.apiKey.trim() : "";
|
||||
const iamToken = typeof cloudSecretConfig.iamToken === "string" ? cloudSecretConfig.iamToken.trim() : "";
|
||||
|
||||
return {
|
||||
provider: "yandex_api",
|
||||
mode: "yandex_search_api",
|
||||
apiBaseUrl: mapped.publicConfig.apiBaseUrl || WORDSTAT_BASE_URL,
|
||||
apiToken: apiKey || iamToken,
|
||||
authType: apiKey ? "api_key" : "iam_token",
|
||||
folderId: cloudPublicConfig.folderId,
|
||||
regionIds: splitCsv(mapped.publicConfig.regionIds, ["225"]),
|
||||
devices: splitCsv(mapped.publicConfig.devices, ["DEVICE_ALL"]),
|
||||
numPhrases: getNumber(mapped.publicConfig.numPhrases, 50)
|
||||
};
|
||||
}
|
||||
|
||||
if (env.marketProviders.wordstat.provider === "mcp_kv" && env.marketProviders.wordstat.mcpEndpoint) {
|
||||
return {
|
||||
provider: "mcp_kv",
|
||||
mode: "mcp_kv",
|
||||
mcpEndpoint: env.marketProviders.wordstat.mcpEndpoint
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
env.marketProviders.wordstat.provider === "yandex_api" &&
|
||||
env.marketProviders.wordstat.apiBaseUrl &&
|
||||
env.marketProviders.wordstat.apiToken &&
|
||||
env.marketProviders.wordstat.folderId
|
||||
) {
|
||||
return {
|
||||
provider: "yandex_api",
|
||||
mode: "yandex_search_api",
|
||||
apiBaseUrl: env.marketProviders.wordstat.apiBaseUrl,
|
||||
apiToken: env.marketProviders.wordstat.apiToken,
|
||||
authType: env.marketProviders.wordstat.authType,
|
||||
folderId: env.marketProviders.wordstat.folderId,
|
||||
regionIds: env.marketProviders.wordstat.regionIds,
|
||||
devices: env.marketProviders.wordstat.devices,
|
||||
numPhrases: env.marketProviders.wordstat.numPhrases
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "disabled",
|
||||
mode: "disabled"
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import { Router } from "express";
|
||||
import type { Response } from "express";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
getExternalServiceSettings,
|
||||
updateExternalServiceSettings,
|
||||
type ExternalServiceId
|
||||
} from "./externalServiceSettings.js";
|
||||
|
||||
export const settingsRouter = Router();
|
||||
|
||||
const externalServiceIdSchema = z.enum([
|
||||
"yandex_cloud",
|
||||
"yandex_metrica",
|
||||
"yandex_search_serp",
|
||||
"yandex_webmaster",
|
||||
"yandex_wordstat"
|
||||
]);
|
||||
|
||||
const updateExternalServiceSettingsSchema = z.object({
|
||||
enabled: z.boolean(),
|
||||
mode: z.string().trim().min(1),
|
||||
publicConfig: z.record(z.string(), z.unknown()).optional(),
|
||||
secrets: z.record(z.string(), z.unknown()).optional()
|
||||
});
|
||||
|
||||
function sendError(response: Response, status: number, message: string) {
|
||||
response.status(status).json({
|
||||
error: {
|
||||
message
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
settingsRouter.get("/external-services", async (_request, response) => {
|
||||
try {
|
||||
response.json({
|
||||
externalServices: await getExternalServiceSettings()
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось загрузить настройки внешних сервисов.");
|
||||
}
|
||||
});
|
||||
|
||||
settingsRouter.patch("/external-services/:serviceId", async (request, response) => {
|
||||
try {
|
||||
const serviceId = externalServiceIdSchema.parse(request.params.serviceId) as ExternalServiceId;
|
||||
const input = updateExternalServiceSettingsSchema.parse(request.body);
|
||||
|
||||
response.json({
|
||||
service: await updateExternalServiceSettings(serviceId, input)
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof z.ZodError) {
|
||||
sendError(response, 400, error.issues[0]?.message ?? "Проверь настройки внешнего сервиса.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.error(error);
|
||||
sendError(response, 500, "Не удалось сохранить настройки внешнего сервиса.");
|
||||
}
|
||||
});
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
import {
|
||||
CopyObjectCommand,
|
||||
CreateBucketCommand,
|
||||
DeleteObjectsCommand,
|
||||
GetObjectCommand,
|
||||
HeadBucketCommand,
|
||||
ListObjectsV2Command,
|
||||
PutObjectCommand,
|
||||
S3Client
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { env } from "../config/env.js";
|
||||
import type { PutObjectInput, StorageObjectSummary, StorageProvider } from "./StorageProvider.js";
|
||||
|
||||
export class S3StorageProvider implements StorageProvider {
|
||||
private readonly client = new S3Client({
|
||||
endpoint: env.s3.endpoint,
|
||||
region: env.s3.region,
|
||||
forcePathStyle: env.s3.forcePathStyle,
|
||||
credentials: {
|
||||
accessKeyId: env.s3.accessKeyId,
|
||||
secretAccessKey: env.s3.secretAccessKey
|
||||
}
|
||||
});
|
||||
|
||||
async ensureReady() {
|
||||
try {
|
||||
await this.client.send(new HeadBucketCommand({ Bucket: env.s3.bucket }));
|
||||
} catch {
|
||||
await this.client.send(new CreateBucketCommand({ Bucket: env.s3.bucket }));
|
||||
}
|
||||
}
|
||||
|
||||
async putObject(input: PutObjectInput) {
|
||||
await this.ensureReady();
|
||||
|
||||
await this.client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: env.s3.bucket,
|
||||
Key: input.key,
|
||||
Body: input.body,
|
||||
ContentType: input.contentType
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async getObject(key: string) {
|
||||
await this.ensureReady();
|
||||
|
||||
const result = await this.client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: env.s3.bucket,
|
||||
Key: key
|
||||
})
|
||||
);
|
||||
const bytes = result.Body ? await result.Body.transformToByteArray() : new Uint8Array();
|
||||
|
||||
return {
|
||||
body: bytes,
|
||||
contentType: result.ContentType
|
||||
};
|
||||
}
|
||||
|
||||
async getObjectText(key: string) {
|
||||
const result = await this.getObject(key);
|
||||
|
||||
return new TextDecoder().decode(result.body);
|
||||
}
|
||||
|
||||
async listPrefix(prefix: string) {
|
||||
let continuationToken: string | undefined;
|
||||
const objects: StorageObjectSummary[] = [];
|
||||
|
||||
await this.ensureReady();
|
||||
|
||||
do {
|
||||
const listedObjects = await this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: env.s3.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
);
|
||||
|
||||
for (const object of listedObjects.Contents ?? []) {
|
||||
if (!object.Key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
objects.push({
|
||||
key: object.Key,
|
||||
size: object.Size ?? 0,
|
||||
lastModified: object.LastModified?.toISOString(),
|
||||
etag: object.ETag
|
||||
});
|
||||
}
|
||||
|
||||
continuationToken = listedObjects.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
async copyPrefix(sourcePrefix: string, destinationPrefix: string) {
|
||||
let continuationToken: string | undefined;
|
||||
let copiedObjects = 0;
|
||||
|
||||
await this.ensureReady();
|
||||
|
||||
do {
|
||||
const listedObjects = await this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: env.s3.bucket,
|
||||
Prefix: sourcePrefix,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
);
|
||||
|
||||
for (const object of listedObjects.Contents ?? []) {
|
||||
if (!object.Key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceKey = object.Key;
|
||||
const destinationKey = `${destinationPrefix}${sourceKey.slice(sourcePrefix.length)}`;
|
||||
|
||||
await this.client.send(
|
||||
new CopyObjectCommand({
|
||||
Bucket: env.s3.bucket,
|
||||
CopySource: `${env.s3.bucket}/${encodeURIComponent(sourceKey).replace(/%2F/g, "/")}`,
|
||||
Key: destinationKey
|
||||
})
|
||||
);
|
||||
copiedObjects += 1;
|
||||
}
|
||||
|
||||
continuationToken = listedObjects.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return copiedObjects;
|
||||
}
|
||||
|
||||
async deletePrefix(prefix: string) {
|
||||
let continuationToken: string | undefined;
|
||||
|
||||
await this.ensureReady();
|
||||
|
||||
do {
|
||||
const listedObjects = await this.client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: env.s3.bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken
|
||||
})
|
||||
);
|
||||
const objects = listedObjects.Contents?.map((object) => ({ Key: object.Key })).filter((object) => object.Key);
|
||||
|
||||
if (objects && objects.length > 0) {
|
||||
await this.client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: env.s3.bucket,
|
||||
Delete: {
|
||||
Objects: objects
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
continuationToken = listedObjects.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
export type PutObjectInput = {
|
||||
key: string;
|
||||
body: string | Uint8Array;
|
||||
contentType?: string;
|
||||
};
|
||||
|
||||
export type StorageObjectSummary = {
|
||||
key: string;
|
||||
size: number;
|
||||
lastModified?: string;
|
||||
etag?: string;
|
||||
};
|
||||
|
||||
export type StorageObject = {
|
||||
body: Uint8Array;
|
||||
contentType?: string;
|
||||
};
|
||||
|
||||
export interface StorageProvider {
|
||||
ensureReady(): Promise<void>;
|
||||
putObject(input: PutObjectInput): Promise<void>;
|
||||
getObject(key: string): Promise<StorageObject>;
|
||||
getObjectText(key: string): Promise<string>;
|
||||
listPrefix(prefix: string): Promise<StorageObjectSummary[]>;
|
||||
copyPrefix(sourcePrefix: string, destinationPrefix: string): Promise<number>;
|
||||
deletePrefix(prefix: string): Promise<void>;
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
import { S3StorageProvider } from "./S3StorageProvider.js";
|
||||
|
||||
export const storageProvider = new S3StorageProvider();
|
||||
|
|
@ -0,0 +1,823 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pool } from "../db/client.js";
|
||||
import { storageProvider } from "../storage/index.js";
|
||||
import { annotatePreviewTargets, getPreviewTargetSelector, getPreviewTargetSeeds } from "./previewTargets.js";
|
||||
|
||||
type PageWorkspaceSourceRow = {
|
||||
project_id: string;
|
||||
page_id: string;
|
||||
scan_version_id: string;
|
||||
source_path: string;
|
||||
url_path: string | null;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
raw: Record<string, unknown>;
|
||||
source_type: string;
|
||||
source_config: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type DraftRow = {
|
||||
id: string;
|
||||
project_id: string;
|
||||
page_id: string;
|
||||
status: string;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
type DraftItemRow = {
|
||||
id: string;
|
||||
draft_id: string;
|
||||
field: string;
|
||||
original_snapshot_key: string | null;
|
||||
working_snapshot_key: string | null;
|
||||
working_text: string | null;
|
||||
created_at: Date;
|
||||
updated_at: Date;
|
||||
};
|
||||
|
||||
type WorkspaceSnapshot = {
|
||||
schemaVersion: "page-workspace.v2";
|
||||
projectId: string;
|
||||
pageId: string;
|
||||
scanVersionId: string;
|
||||
sourcePath: string;
|
||||
urlPath: string | null;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
headings: Array<{ level: number; text: string }>;
|
||||
originalText: string;
|
||||
originalHtml: string;
|
||||
previewHtml: string;
|
||||
sections: PageWorkspaceSection[];
|
||||
createdAt: string;
|
||||
};
|
||||
|
||||
export type PageWorkspaceSection = {
|
||||
id: string;
|
||||
kind: "meta" | "content_block" | "heading_group" | "media";
|
||||
title: string;
|
||||
selector: string | null;
|
||||
originalText: string;
|
||||
workingText: string;
|
||||
charCount: number;
|
||||
summary: string;
|
||||
sourceHint: string;
|
||||
};
|
||||
|
||||
export type PageWorkspace = {
|
||||
projectId: string;
|
||||
pageId: string;
|
||||
page: {
|
||||
scanVersionId: string;
|
||||
sourcePath: string;
|
||||
urlPath: string | null;
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
headings: Array<{ level: number; text: string }>;
|
||||
};
|
||||
draft: {
|
||||
id: string;
|
||||
status: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
};
|
||||
item: {
|
||||
id: string;
|
||||
field: string;
|
||||
originalSnapshotKey: string | null;
|
||||
workingSnapshotKey: string | null;
|
||||
originalText: string;
|
||||
workingText: string;
|
||||
originalTextLength: number;
|
||||
workingTextLength: number;
|
||||
updatedAt: string;
|
||||
};
|
||||
previewHtml: string;
|
||||
sections: PageWorkspaceSection[];
|
||||
};
|
||||
|
||||
export class PageWorkspaceError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "PageWorkspaceError";
|
||||
}
|
||||
}
|
||||
|
||||
const WORKSPACE_FIELD = "page_text";
|
||||
const MAX_BLOCK_TEXT_LENGTH = 6_000;
|
||||
const MAX_SECTIONS = 80;
|
||||
const MAX_MEDIA_SECTIONS = 80;
|
||||
|
||||
function toIsoDate(value: Date) {
|
||||
return value.toISOString();
|
||||
}
|
||||
|
||||
function normalizePath(value: string) {
|
||||
return value.replace(/\\/g, "/").replace(/^\/+/, "");
|
||||
}
|
||||
|
||||
function getStringConfig(config: Record<string, unknown>, key: string) {
|
||||
const value = config[key];
|
||||
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string) {
|
||||
return value
|
||||
.replace(/ /gi, " ")
|
||||
.replace(/&/gi, "&")
|
||||
.replace(/"/gi, "\"")
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/</gi, "<")
|
||||
.replace(/>/gi, ">");
|
||||
}
|
||||
|
||||
function normalizeWhitespace(value: string) {
|
||||
return decodeHtmlEntities(value).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function stripIgnoredHtmlBlocks(html: string) {
|
||||
return html
|
||||
.replace(/<script\b[\s\S]*?<\/script>/gi, " ")
|
||||
.replace(/<style\b[\s\S]*?<\/style>/gi, " ")
|
||||
.replace(/<svg\b[\s\S]*?<\/svg>/gi, " ");
|
||||
}
|
||||
|
||||
function getBodyHtml(html: string) {
|
||||
const bodyMatch = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(html);
|
||||
return bodyMatch?.[1] ?? html;
|
||||
}
|
||||
|
||||
function getAttribute(attributes: string, name: string) {
|
||||
const pattern = new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i");
|
||||
return pattern.exec(attributes)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function getDisplayFilename(source: string | null, fallback: string) {
|
||||
if (!source) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
const cleanSource = source.split(/[?#]/)[0] ?? source;
|
||||
return path.basename(cleanSource) || source;
|
||||
}
|
||||
|
||||
function htmlToReadableText(html: string) {
|
||||
const withBreaks = stripIgnoredHtmlBlocks(html)
|
||||
.replace(/<br\s*\/?>/gi, "\n")
|
||||
.replace(/<\/(p|div|section|article|header|footer|main|aside|nav|li|ul|ol|h[1-6])>/gi, "\n")
|
||||
.replace(/<(h[1-6])\b[^>]*>/gi, "\n");
|
||||
|
||||
return decodeHtmlEntities(withBreaks.replace(/<[^>]+>/g, " "))
|
||||
.split(/\n+/)
|
||||
.map((line) => normalizeWhitespace(line))
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.slice(0, MAX_BLOCK_TEXT_LENGTH);
|
||||
}
|
||||
|
||||
function normalizeHeadings(raw: Record<string, unknown>) {
|
||||
const headings = raw.headings;
|
||||
|
||||
if (!Array.isArray(headings)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return headings
|
||||
.map((heading) => {
|
||||
if (typeof heading !== "object" || heading === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const level = "level" in heading && typeof heading.level === "number" ? heading.level : null;
|
||||
const text = "text" in heading && typeof heading.text === "string" ? heading.text : null;
|
||||
|
||||
return level && text ? { level, text } : null;
|
||||
})
|
||||
.filter((heading): heading is { level: number; text: string } => Boolean(heading));
|
||||
}
|
||||
|
||||
function getFirstH1Text(raw: Record<string, unknown>) {
|
||||
return normalizeHeadings(raw).find((heading) => heading.level === 1)?.text ?? "";
|
||||
}
|
||||
|
||||
function countImageTags(html: string) {
|
||||
return Array.from(getBodyHtml(html).matchAll(/<img\b[^>]*>/gi)).length;
|
||||
}
|
||||
|
||||
function getSectionSummary(text: string) {
|
||||
const firstMeaningfulLine = text
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line.length > 0);
|
||||
|
||||
if (!firstMeaningfulLine) {
|
||||
return "Пустой текстовый блок.";
|
||||
}
|
||||
|
||||
return firstMeaningfulLine.length > 180 ? `${firstMeaningfulLine.slice(0, 177)}...` : firstMeaningfulLine;
|
||||
}
|
||||
|
||||
function makeSection(input: Omit<PageWorkspaceSection, "charCount" | "summary" | "workingText">): PageWorkspaceSection {
|
||||
return {
|
||||
...input,
|
||||
workingText: input.originalText,
|
||||
charCount: input.originalText.length,
|
||||
summary: getSectionSummary(input.originalText)
|
||||
};
|
||||
}
|
||||
|
||||
function getFirstHeadingText(html: string) {
|
||||
const headingMatch = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/i.exec(html);
|
||||
return headingMatch ? htmlToReadableText(headingMatch[2] ?? "") : null;
|
||||
}
|
||||
|
||||
function getSemanticBlockTitle(tagName: string, attributes: string, text: string, fallbackIndex: number) {
|
||||
const heading = getFirstHeadingText(text);
|
||||
const id = getAttribute(attributes, "id");
|
||||
const className = getAttribute(attributes, "class");
|
||||
const label = getAttribute(attributes, "aria-label");
|
||||
|
||||
if (heading) return heading;
|
||||
if (label) return label;
|
||||
if (id) return `${tagName}#${id}`;
|
||||
if (className) return `${tagName}.${className.split(/\s+/).filter(Boolean).slice(0, 2).join(".")}`;
|
||||
return `Блок ${fallbackIndex}`;
|
||||
}
|
||||
|
||||
function dedupeSections(sections: PageWorkspaceSection[]) {
|
||||
const seenTexts = new Set<string>();
|
||||
const dedupedSections: PageWorkspaceSection[] = [];
|
||||
|
||||
for (const section of sections) {
|
||||
const normalizedText = normalizeWhitespace(section.originalText).toLocaleLowerCase();
|
||||
|
||||
if (!normalizedText || seenTexts.has(normalizedText)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
seenTexts.add(normalizedText);
|
||||
dedupedSections.push(section);
|
||||
}
|
||||
|
||||
return dedupedSections.slice(0, MAX_SECTIONS);
|
||||
}
|
||||
|
||||
function extractSemanticSections(html: string) {
|
||||
const bodyHtml = getBodyHtml(html);
|
||||
const blockPattern = /<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi;
|
||||
const sections: PageWorkspaceSection[] = [];
|
||||
let match: RegExpExecArray | null;
|
||||
let index = 1;
|
||||
|
||||
while ((match = blockPattern.exec(bodyHtml)) && sections.length < MAX_SECTIONS) {
|
||||
const tagName = match[1] ?? "section";
|
||||
const attributes = match[2] ?? "";
|
||||
const blockHtml = match[3] ?? "";
|
||||
const text = htmlToReadableText(blockHtml);
|
||||
|
||||
if (text.length < 20) {
|
||||
continue;
|
||||
}
|
||||
|
||||
sections.push(
|
||||
makeSection({
|
||||
id: `block-${index}`,
|
||||
kind: "content_block",
|
||||
title: getSemanticBlockTitle(tagName, attributes, blockHtml, index),
|
||||
selector: getPreviewTargetSelector(`block-${index}`),
|
||||
originalText: text,
|
||||
sourceHint: `<${tagName}>`
|
||||
})
|
||||
);
|
||||
index += 1;
|
||||
}
|
||||
|
||||
return dedupeSections(sections);
|
||||
}
|
||||
|
||||
function extractHeadingSections(html: string) {
|
||||
const bodyHtml = getBodyHtml(html);
|
||||
const headingPattern = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
|
||||
const headingMatches = Array.from(bodyHtml.matchAll(headingPattern));
|
||||
const sections: PageWorkspaceSection[] = [];
|
||||
|
||||
headingMatches.forEach((headingMatch, index) => {
|
||||
if (sections.length >= MAX_SECTIONS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const level = headingMatch[1] ?? "";
|
||||
const headingHtml = headingMatch[2] ?? "";
|
||||
const startIndex = headingMatch.index ?? 0;
|
||||
const nextIndex = headingMatches[index + 1]?.index ?? bodyHtml.length;
|
||||
const chunkHtml = bodyHtml.slice(startIndex, nextIndex);
|
||||
const text = htmlToReadableText(chunkHtml);
|
||||
const headingText = htmlToReadableText(headingHtml) || `H${level}`;
|
||||
|
||||
if (text.length < 8) {
|
||||
return;
|
||||
}
|
||||
|
||||
sections.push(
|
||||
makeSection({
|
||||
id: `heading-group-${index + 1}`,
|
||||
kind: "heading_group",
|
||||
title: `H${level}: ${headingText}`,
|
||||
selector: getPreviewTargetSelector(`heading-group-${index + 1}`),
|
||||
originalText: text,
|
||||
sourceHint: "heading-группа"
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
return dedupeSections(sections);
|
||||
}
|
||||
|
||||
function extractImageAltSections(html: string) {
|
||||
const bodyHtml = getBodyHtml(html);
|
||||
const imagePattern = /<img\b([^>]*)>/gi;
|
||||
const sections: PageWorkspaceSection[] = [];
|
||||
let match: RegExpExecArray | null;
|
||||
let imageIndex = 0;
|
||||
|
||||
while ((match = imagePattern.exec(bodyHtml)) && sections.length < MAX_MEDIA_SECTIONS) {
|
||||
imageIndex += 1;
|
||||
|
||||
const attributes = match[1] ?? "";
|
||||
const source = getAttribute(attributes, "src");
|
||||
const alt = getAttribute(attributes, "alt");
|
||||
const normalizedAlt = normalizeWhitespace(alt ?? "");
|
||||
|
||||
const filename = getDisplayFilename(source, `image-${imageIndex}`);
|
||||
|
||||
sections.push(
|
||||
makeSection({
|
||||
id: `media-alt-${imageIndex}`,
|
||||
kind: "media",
|
||||
title: `ALT: ${filename}`,
|
||||
selector: getPreviewTargetSelector(`media-alt-${imageIndex}`),
|
||||
originalText: normalizedAlt,
|
||||
sourceHint: source ?? `img:nth-of-type(${imageIndex})`
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return sections;
|
||||
}
|
||||
|
||||
function buildPreviewHtml(html: string) {
|
||||
const previewStyles = `
|
||||
<style>
|
||||
html { background: #fff !important; }
|
||||
body { min-height: 100vh; }
|
||||
a { cursor: default !important; }
|
||||
</style>
|
||||
`;
|
||||
let imageIndex = 0;
|
||||
const targetSeeds = getPreviewTargetSeeds(html);
|
||||
const withoutScripts = annotatePreviewTargets(html, targetSeeds)
|
||||
.replace(/<script\b[\s\S]*?<\/script>/gi, "")
|
||||
.replace(/\son[a-z]+\s*=\s*["'][^"']*["']/gi, "")
|
||||
.replace(/<h([1-3])\b([^>]*)>/gi, (_match, level: string, attributes: string) => {
|
||||
return `<h${level}${attributes} data-seo-label="H${level}">`;
|
||||
})
|
||||
.replace(/<img\b([^>]*)>/gi, (match, attributes: string) => {
|
||||
imageIndex += 1;
|
||||
|
||||
if (/data-seo-media-index\s*=/i.test(attributes)) {
|
||||
return match;
|
||||
}
|
||||
|
||||
const isSelfClosing = /\/\s*$/.test(attributes);
|
||||
const cleanAttributes = attributes.replace(/\/\s*$/, "").trimEnd();
|
||||
return `<img${cleanAttributes} data-seo-media-index="${imageIndex}"${isSelfClosing ? " /" : ""}>`;
|
||||
});
|
||||
|
||||
if (/<head\b[^>]*>/i.test(withoutScripts)) {
|
||||
return withoutScripts.replace(/<head\b[^>]*>/i, (headTag) => `${headTag}<base target="_blank">${previewStyles}`);
|
||||
}
|
||||
|
||||
return `<!doctype html><html><head><base target="_blank">${previewStyles}</head><body>${withoutScripts}</body></html>`;
|
||||
}
|
||||
|
||||
function buildSections(page: PageWorkspaceSourceRow, html: string): PageWorkspaceSection[] {
|
||||
const sections: PageWorkspaceSection[] = [];
|
||||
|
||||
if (page.title) {
|
||||
sections.push(makeSection({
|
||||
id: "seo-title",
|
||||
kind: "meta",
|
||||
title: "Title",
|
||||
selector: "head > title",
|
||||
originalText: page.title,
|
||||
sourceHint: "SEO-поле"
|
||||
}));
|
||||
}
|
||||
|
||||
if (page.description) {
|
||||
sections.push(makeSection({
|
||||
id: "seo-description",
|
||||
kind: "meta",
|
||||
title: "Meta description",
|
||||
selector: "meta[name='description']",
|
||||
originalText: page.description,
|
||||
sourceHint: "SEO-поле"
|
||||
}));
|
||||
}
|
||||
|
||||
const semanticSections = extractSemanticSections(html);
|
||||
const contentSections = semanticSections.length >= 2 ? semanticSections : extractHeadingSections(html);
|
||||
const mediaSections = extractImageAltSections(html);
|
||||
|
||||
return [...sections, ...contentSections, ...mediaSections];
|
||||
}
|
||||
|
||||
function buildIssueAlignedSections(page: PageWorkspaceSourceRow, html: string): PageWorkspaceSection[] {
|
||||
const semanticSections = extractSemanticSections(html);
|
||||
const contentSections = semanticSections.length >= 2 ? semanticSections : extractHeadingSections(html);
|
||||
const mediaSections = extractImageAltSections(html);
|
||||
const pageAnchorSelector = contentSections[0]?.selector ?? mediaSections[0]?.selector ?? getPreviewTargetSelector("block-1");
|
||||
const h1Section = contentSections.find((section) => section.title.toLocaleLowerCase().startsWith("h1:"));
|
||||
const seoSections: PageWorkspaceSection[] = [
|
||||
makeSection({
|
||||
id: "seo-title",
|
||||
kind: "meta",
|
||||
title: "Title",
|
||||
selector: pageAnchorSelector,
|
||||
originalText: page.title ?? "",
|
||||
sourceHint: "<title>"
|
||||
}),
|
||||
makeSection({
|
||||
id: "seo-description",
|
||||
kind: "meta",
|
||||
title: "Meta description",
|
||||
selector: pageAnchorSelector,
|
||||
originalText: page.description ?? "",
|
||||
sourceHint: "meta[name='description']"
|
||||
}),
|
||||
makeSection({
|
||||
id: "seo-h1",
|
||||
kind: "heading_group",
|
||||
title: "H1",
|
||||
selector: h1Section?.selector ?? pageAnchorSelector,
|
||||
originalText: getFirstH1Text(page.raw),
|
||||
sourceHint: "H1"
|
||||
})
|
||||
];
|
||||
|
||||
return [...seoSections, ...contentSections, ...mediaSections];
|
||||
}
|
||||
|
||||
function buildOriginalText(page: PageWorkspaceSourceRow, sections: PageWorkspaceSection[]) {
|
||||
const header = [
|
||||
`Источник: ${page.source_path}`,
|
||||
page.url_path ? `URL: ${page.url_path}` : null,
|
||||
`Scan version: ${page.scan_version_id}`
|
||||
].filter(Boolean);
|
||||
|
||||
const sectionText = sections.map((section) => {
|
||||
return `## ${section.title}\n${section.originalText}`;
|
||||
});
|
||||
|
||||
return [...header, "", ...sectionText].join("\n\n").trim();
|
||||
}
|
||||
|
||||
function extractWorkingSectionTexts(workingText: string, sectionCount: number) {
|
||||
const headings = Array.from(workingText.matchAll(/^## .+$/gm));
|
||||
|
||||
if (headings.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return headings.slice(0, sectionCount).map((heading, index) => {
|
||||
const start = (heading.index ?? 0) + heading[0].length;
|
||||
const end = headings[index + 1]?.index ?? workingText.length;
|
||||
return workingText.slice(start, end).trim();
|
||||
});
|
||||
}
|
||||
|
||||
async function getLatestPageWorkspaceSource(projectId: string, pageId: string) {
|
||||
const result = await pool.query<PageWorkspaceSourceRow>(
|
||||
`
|
||||
select
|
||||
sv.project_id,
|
||||
pv.page_id,
|
||||
pv.scan_version_id,
|
||||
pv.source_path,
|
||||
pv.url_path,
|
||||
pv.title,
|
||||
pv.description,
|
||||
pv.raw,
|
||||
ps.type as source_type,
|
||||
ps.config as source_config
|
||||
from page_versions pv
|
||||
join scan_versions sv on sv.id = pv.scan_version_id
|
||||
join project_sources ps on ps.id = sv.source_id
|
||||
where sv.project_id = $1
|
||||
and pv.page_id = $2
|
||||
and sv.status = 'done'
|
||||
order by sv.completed_at desc nulls last, sv.created_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId, pageId]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function readPageHtml(page: PageWorkspaceSourceRow) {
|
||||
if (page.source_type === "browser_folder") {
|
||||
const importPrefix = getStringConfig(page.source_config, "importPrefix");
|
||||
|
||||
if (!importPrefix) {
|
||||
throw new PageWorkspaceError("У проекта нет importPrefix для чтения HTML из MinIO.");
|
||||
}
|
||||
|
||||
const normalizedPrefix = importPrefix.endsWith("/") ? importPrefix : `${importPrefix}/`;
|
||||
return storageProvider.getObjectText(`${normalizedPrefix}${normalizePath(page.source_path)}`);
|
||||
}
|
||||
|
||||
if (page.source_type === "local_folder") {
|
||||
const sourcePath = getStringConfig(page.source_config, "path");
|
||||
|
||||
if (!sourcePath) {
|
||||
throw new PageWorkspaceError("У проекта нет локального пути для чтения HTML.");
|
||||
}
|
||||
|
||||
return fs.readFile(path.join(sourcePath, normalizePath(page.source_path)), "utf8");
|
||||
}
|
||||
|
||||
throw new PageWorkspaceError("Рабочая зона для этого типа источника пока не поддерживается.");
|
||||
}
|
||||
|
||||
async function getActiveDraft(projectId: string, pageId: string) {
|
||||
const result = await pool.query<DraftRow>(
|
||||
`
|
||||
select *
|
||||
from drafts
|
||||
where project_id = $1 and page_id = $2 and status = 'active'
|
||||
order by updated_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[projectId, pageId]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function getDraftItem(draftId: string) {
|
||||
const result = await pool.query<DraftItemRow>(
|
||||
`
|
||||
select *
|
||||
from draft_items
|
||||
where draft_id = $1 and field = $2
|
||||
order by updated_at desc
|
||||
limit 1;
|
||||
`,
|
||||
[draftId, WORKSPACE_FIELD]
|
||||
);
|
||||
|
||||
return result.rows[0] ?? null;
|
||||
}
|
||||
|
||||
async function readOriginalSnapshot(key: string | null) {
|
||||
if (!key) {
|
||||
throw new PageWorkspaceError("У draft нет original snapshot.");
|
||||
}
|
||||
|
||||
const text = await storageProvider.getObjectText(key);
|
||||
return JSON.parse(text) as WorkspaceSnapshot;
|
||||
}
|
||||
|
||||
function buildWorkspaceResponse(page: PageWorkspaceSourceRow, draft: DraftRow, item: DraftItemRow, snapshot: WorkspaceSnapshot): PageWorkspace {
|
||||
const workingText = item.working_text ?? snapshot.originalText;
|
||||
const workingSectionTexts = extractWorkingSectionTexts(workingText, snapshot.sections.length);
|
||||
const sections = snapshot.sections.map((section, index) => {
|
||||
const sectionWorkingText = workingSectionTexts[index] ?? section.originalText;
|
||||
|
||||
return {
|
||||
...section,
|
||||
workingText: sectionWorkingText,
|
||||
charCount: sectionWorkingText.length,
|
||||
summary: getSectionSummary(sectionWorkingText)
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
projectId: page.project_id,
|
||||
pageId: page.page_id,
|
||||
page: {
|
||||
scanVersionId: page.scan_version_id,
|
||||
sourcePath: page.source_path,
|
||||
urlPath: page.url_path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
headings: normalizeHeadings(page.raw)
|
||||
},
|
||||
draft: {
|
||||
id: draft.id,
|
||||
status: draft.status,
|
||||
createdAt: toIsoDate(draft.created_at),
|
||||
updatedAt: toIsoDate(draft.updated_at)
|
||||
},
|
||||
item: {
|
||||
id: item.id,
|
||||
field: item.field,
|
||||
originalSnapshotKey: item.original_snapshot_key,
|
||||
workingSnapshotKey: item.working_snapshot_key,
|
||||
originalText: snapshot.originalText,
|
||||
workingText,
|
||||
originalTextLength: snapshot.originalText.length,
|
||||
workingTextLength: workingText.length,
|
||||
updatedAt: toIsoDate(item.updated_at)
|
||||
},
|
||||
previewHtml: snapshot.previewHtml,
|
||||
sections
|
||||
};
|
||||
}
|
||||
|
||||
function buildSnapshot(page: PageWorkspaceSourceRow, html: string): WorkspaceSnapshot {
|
||||
const sections = buildIssueAlignedSections(page, html);
|
||||
const originalText = buildOriginalText(page, sections);
|
||||
|
||||
return {
|
||||
schemaVersion: "page-workspace.v2",
|
||||
projectId: page.project_id,
|
||||
pageId: page.page_id,
|
||||
scanVersionId: page.scan_version_id,
|
||||
sourcePath: page.source_path,
|
||||
urlPath: page.url_path,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
headings: normalizeHeadings(page.raw),
|
||||
originalText,
|
||||
originalHtml: html,
|
||||
previewHtml: buildPreviewHtml(html),
|
||||
sections,
|
||||
createdAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
async function getUpgradedSnapshot(page: PageWorkspaceSourceRow, item: DraftItemRow) {
|
||||
const snapshot = await readOriginalSnapshot(item.original_snapshot_key);
|
||||
const imageCount = Math.min(countImageTags(snapshot.originalHtml ?? ""), MAX_MEDIA_SECTIONS);
|
||||
const mediaSectionCount = snapshot.sections.filter((section) => section.kind === "media").length;
|
||||
const isCurrentSnapshot =
|
||||
snapshot.schemaVersion === "page-workspace.v2" &&
|
||||
"previewHtml" in snapshot &&
|
||||
snapshot.sections.some((section) => section.id === "seo-title") &&
|
||||
snapshot.sections.some((section) => section.id === "seo-description") &&
|
||||
snapshot.sections.some((section) => section.id === "seo-h1") &&
|
||||
snapshot.sections.some((section) => section.kind === "content_block" || section.kind === "heading_group") &&
|
||||
(!/<img\b/i.test(snapshot.originalHtml ?? "") || mediaSectionCount >= imageCount) &&
|
||||
snapshot.sections.every((section) => section.kind === "meta" || section.selector?.includes("data-seo-target-id"));
|
||||
|
||||
if (isCurrentSnapshot) {
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
const html = snapshot.originalHtml || (await readPageHtml(page));
|
||||
const upgradedSnapshot = buildSnapshot(page, html);
|
||||
|
||||
if (item.original_snapshot_key) {
|
||||
await storageProvider.putObject({
|
||||
key: item.original_snapshot_key,
|
||||
body: JSON.stringify(upgradedSnapshot, null, 2),
|
||||
contentType: "application/json"
|
||||
});
|
||||
}
|
||||
|
||||
if (!item.working_text || item.working_text === snapshot.originalText) {
|
||||
await pool.query(
|
||||
`
|
||||
update draft_items
|
||||
set working_text = $1, updated_at = now()
|
||||
where id = $2;
|
||||
`,
|
||||
[upgradedSnapshot.originalText, item.id]
|
||||
);
|
||||
item.working_text = upgradedSnapshot.originalText;
|
||||
item.updated_at = new Date();
|
||||
}
|
||||
|
||||
return upgradedSnapshot;
|
||||
}
|
||||
|
||||
export async function getOrCreatePageWorkspace(projectId: string, pageId: string) {
|
||||
const page = await getLatestPageWorkspaceSource(projectId, pageId);
|
||||
|
||||
if (!page) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const existingDraft = await getActiveDraft(projectId, pageId);
|
||||
const existingItem = existingDraft ? await getDraftItem(existingDraft.id) : null;
|
||||
|
||||
if (existingDraft && existingItem) {
|
||||
const snapshot = await getUpgradedSnapshot(page, existingItem);
|
||||
return buildWorkspaceResponse(page, existingDraft, existingItem, snapshot);
|
||||
}
|
||||
|
||||
const html = await readPageHtml(page);
|
||||
const snapshot = buildSnapshot(page, html);
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
await client.query("begin");
|
||||
|
||||
const draftResult = await client.query<DraftRow>(
|
||||
`
|
||||
insert into drafts (project_id, page_id, status)
|
||||
values ($1, $2, 'active')
|
||||
returning *;
|
||||
`,
|
||||
[projectId, pageId]
|
||||
);
|
||||
const draft = draftResult.rows[0];
|
||||
|
||||
if (!draft) {
|
||||
throw new Error("Не удалось создать draft.");
|
||||
}
|
||||
|
||||
const originalSnapshotKey = `projects/${projectId}/drafts/${draft.id}/original-page-workspace.json`;
|
||||
await storageProvider.putObject({
|
||||
key: originalSnapshotKey,
|
||||
body: JSON.stringify(snapshot, null, 2),
|
||||
contentType: "application/json"
|
||||
});
|
||||
|
||||
const itemResult = await client.query<DraftItemRow>(
|
||||
`
|
||||
insert into draft_items (draft_id, field, original_snapshot_key, working_text)
|
||||
values ($1, $2, $3, $4)
|
||||
returning *;
|
||||
`,
|
||||
[draft.id, WORKSPACE_FIELD, originalSnapshotKey, snapshot.originalText]
|
||||
);
|
||||
const item = itemResult.rows[0];
|
||||
|
||||
if (!item) {
|
||||
throw new Error("Не удалось создать draft item.");
|
||||
}
|
||||
|
||||
await client.query("commit");
|
||||
return buildWorkspaceResponse(page, draft, item, snapshot);
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function updateWorkspaceText(projectId: string, pageId: string, workingText: string) {
|
||||
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
|
||||
|
||||
if (!workspace) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const workingSnapshotKey = `projects/${projectId}/drafts/${workspace.draft.id}/working-${Date.now()}.txt`;
|
||||
|
||||
await storageProvider.putObject({
|
||||
key: workingSnapshotKey,
|
||||
body: workingText,
|
||||
contentType: "text/plain; charset=utf-8"
|
||||
});
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
update draft_items
|
||||
set working_text = $1, working_snapshot_key = $2, updated_at = now()
|
||||
where id = $3;
|
||||
`,
|
||||
[workingText, workingSnapshotKey, workspace.item.id]
|
||||
);
|
||||
|
||||
await pool.query(
|
||||
`
|
||||
update drafts
|
||||
set updated_at = now()
|
||||
where id = $1;
|
||||
`,
|
||||
[workspace.draft.id]
|
||||
);
|
||||
|
||||
return getOrCreatePageWorkspace(projectId, pageId);
|
||||
}
|
||||
|
||||
export async function savePageWorkspace(projectId: string, pageId: string, workingText: string) {
|
||||
return updateWorkspaceText(projectId, pageId, workingText);
|
||||
}
|
||||
|
||||
export async function resetPageWorkspace(projectId: string, pageId: string) {
|
||||
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
|
||||
|
||||
if (!workspace) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return updateWorkspaceText(projectId, pageId, workspace.item.originalText);
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
export type PreviewTargetSeed = {
|
||||
id: string;
|
||||
kind: "content_block" | "heading_group" | "media";
|
||||
selector: string;
|
||||
sourceHint: string;
|
||||
targetIndex: number;
|
||||
};
|
||||
|
||||
function normalizeWhitespace(value: string) {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function getBodyHtml(html: string) {
|
||||
const bodyMatch = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(html);
|
||||
return bodyMatch?.[1] ?? html;
|
||||
}
|
||||
|
||||
function getAttribute(attributes: string, name: string) {
|
||||
const pattern = new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i");
|
||||
return pattern.exec(attributes)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function htmlToPlainText(html: string) {
|
||||
return normalizeWhitespace(html.replace(/<script\b[\s\S]*?<\/script>/gi, " ").replace(/<style\b[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " "));
|
||||
}
|
||||
|
||||
function escapeAttribute(value: string) {
|
||||
return value.replace(/&/g, "&").replace(/"/g, """);
|
||||
}
|
||||
|
||||
function injectAttribute(openingTag: string, name: string, value: string) {
|
||||
if (new RegExp(`\\s${name}\\s*=`, "i").test(openingTag)) {
|
||||
return openingTag;
|
||||
}
|
||||
|
||||
return openingTag.replace(/\/?\s*>$/, (ending) => ` ${name}="${escapeAttribute(value)}"${ending}`);
|
||||
}
|
||||
|
||||
export function getPreviewTargetSelector(targetId: string) {
|
||||
return `[data-seo-target-id="${targetId}"]`;
|
||||
}
|
||||
|
||||
export function getPreviewTargetSeeds(html: string) {
|
||||
const bodyHtml = getBodyHtml(html);
|
||||
const seeds: PreviewTargetSeed[] = [];
|
||||
const blockPattern = /<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi;
|
||||
let blockMatch: RegExpExecArray | null;
|
||||
let blockIndex = 1;
|
||||
|
||||
while ((blockMatch = blockPattern.exec(bodyHtml)) && seeds.length < 80) {
|
||||
const tagName = blockMatch[1] ?? "section";
|
||||
const blockHtml = blockMatch[3] ?? "";
|
||||
const text = htmlToPlainText(blockHtml);
|
||||
|
||||
if (text.length < 20) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const id = `block-${blockIndex}`;
|
||||
seeds.push({
|
||||
id,
|
||||
kind: "content_block",
|
||||
selector: getPreviewTargetSelector(id),
|
||||
sourceHint: `<${tagName}>`,
|
||||
targetIndex: blockIndex
|
||||
});
|
||||
blockIndex += 1;
|
||||
}
|
||||
|
||||
const headingPattern = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
|
||||
let headingMatch: RegExpExecArray | null;
|
||||
let headingIndex = 1;
|
||||
|
||||
while ((headingMatch = headingPattern.exec(bodyHtml)) && seeds.length < 120) {
|
||||
const text = htmlToPlainText(headingMatch[2] ?? "");
|
||||
|
||||
if (text.length < 2) {
|
||||
headingIndex += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const id = `heading-group-${headingIndex}`;
|
||||
seeds.push({
|
||||
id,
|
||||
kind: "heading_group",
|
||||
selector: getPreviewTargetSelector(id),
|
||||
sourceHint: `H${headingMatch[1] ?? ""}`,
|
||||
targetIndex: headingIndex
|
||||
});
|
||||
headingIndex += 1;
|
||||
}
|
||||
|
||||
const imagePattern = /<img\b([^>]*)>/gi;
|
||||
let imageMatch: RegExpExecArray | null;
|
||||
let imageIndex = 0;
|
||||
while ((imageMatch = imagePattern.exec(bodyHtml)) && seeds.length < 180) {
|
||||
imageIndex += 1;
|
||||
|
||||
const attributes = imageMatch[1] ?? "";
|
||||
const id = `media-alt-${imageIndex}`;
|
||||
seeds.push({
|
||||
id,
|
||||
kind: "media",
|
||||
selector: getPreviewTargetSelector(id),
|
||||
sourceHint: getAttribute(attributes, "src") ?? `img:nth-of-type(${imageIndex})`,
|
||||
targetIndex: imageIndex
|
||||
});
|
||||
}
|
||||
|
||||
return seeds;
|
||||
}
|
||||
|
||||
export function annotatePreviewTargets(html: string, seeds: PreviewTargetSeed[]) {
|
||||
const blockSeeds = seeds.filter((seed) => seed.kind === "content_block");
|
||||
const headingSeeds = seeds.filter((seed) => seed.kind === "heading_group");
|
||||
const mediaSeeds = seeds.filter((seed) => seed.kind === "media");
|
||||
let blockIndex = 0;
|
||||
let headingIndex = 0;
|
||||
let imageIndex = 0;
|
||||
|
||||
return html
|
||||
.replace(/<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi, (blockHtml, tagName: string, attributes: string, content: string) => {
|
||||
const text = htmlToPlainText(content);
|
||||
|
||||
if (text.length < 20) {
|
||||
return blockHtml;
|
||||
}
|
||||
|
||||
const seed = blockSeeds[blockIndex];
|
||||
blockIndex += 1;
|
||||
|
||||
if (!seed) {
|
||||
return blockHtml;
|
||||
}
|
||||
|
||||
const openingTag = `<${tagName}${attributes}>`;
|
||||
return blockHtml.replace(openingTag, injectAttribute(openingTag, "data-seo-target-id", seed.id));
|
||||
})
|
||||
.replace(/<h([1-6])\b([^>]*)>([\s\S]*?)<\/h\1>/gi, (headingHtml, level: string, attributes: string, content: string) => {
|
||||
const text = htmlToPlainText(content);
|
||||
|
||||
if (text.length < 2) {
|
||||
return headingHtml;
|
||||
}
|
||||
|
||||
const seed = headingSeeds[headingIndex];
|
||||
headingIndex += 1;
|
||||
|
||||
if (!seed) {
|
||||
return headingHtml;
|
||||
}
|
||||
|
||||
const openingTag = `<h${level}${attributes}>`;
|
||||
return headingHtml.replace(openingTag, injectAttribute(openingTag, "data-seo-target-id", seed.id));
|
||||
})
|
||||
.replace(/<img\b([^>]*)>/gi, (openingTag) => {
|
||||
imageIndex += 1;
|
||||
const seed = mediaSeeds.find((candidate) => candidate.targetIndex === imageIndex);
|
||||
|
||||
return seed ? injectAttribute(openingTag, "data-seo-target-id", seed.id) : openingTag;
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"extends": "../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"types": ["node"],
|
||||
"noEmitOnError": true
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"useDefineForClassFields": true,
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue