Add SEO mode project workflow

This commit is contained in:
DCCONSTRUCTIONS 2026-06-28 12:05:49 +03:00
parent 20cf1a40f0
commit d27e6ee43d
60 changed files with 36186 additions and 0 deletions

36
.gitignore vendored Normal file
View File

@ -0,0 +1,36 @@
# Dependencies
node_modules/
# Builds
dist/
build/
.vite/
# Environment and secrets
.env
.env.*
!.env.example
# Local runtime data
data/
tmp/
temp/
logs/
*.log
# Local infrastructure volumes
.postgres/
.minio/
# Unpacked archive references kept local; source archives stay tracked.
/SEO-farm/
/seo_mode_test_site/
# Nested VCS metadata from unpacked archives
.git/
# OS / editor
.DS_Store
Thumbs.db
.idea/
.vscode/

View File

@ -0,0 +1,8 @@
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_size = 2

View File

@ -0,0 +1,27 @@
SERVER_PORT=4100
DATABASE_URL=postgres://seo_farm:seo_farm@localhost:5432/seo_farm
S3_ENDPOINT=http://localhost:9000
S3_REGION=us-east-1
S3_ACCESS_KEY_ID=seo_farm
S3_SECRET_ACCESS_KEY=seo_farm_secret
S3_BUCKET=seo-farm-artifacts
S3_FORCE_PATH_STYLE=true
WORDSTAT_PROVIDER=disabled
# WORDSTAT_PROVIDER=mcp_kv
# WORDSTAT_MCP_ENDPOINT=http://localhost:4120/wordstat/collect
# WORDSTAT_PROVIDER=yandex_api
# WORDSTAT_API_BASE_URL=https://searchapi.api.cloud.yandex.net/v2/wordstat
# WORDSTAT_API_TOKEN=change-me
# WORDSTAT_AUTH_TYPE=api_key
# WORDSTAT_FOLDER_ID=b1g...
# WORDSTAT_REGION_IDS=225
# WORDSTAT_DEVICES=DEVICE_ALL
# WORDSTAT_NUM_PHRASES=50
SERP_PROVIDER=disabled
# SERP_API_BASE_URL=https://serp-provider.internal
# SERP_API_TOKEN=change-me
VITE_API_BASE_URL=http://localhost:4100

29
seo_mode/seo_mode/.gitignore vendored Normal file
View File

@ -0,0 +1,29 @@
# Dependencies
node_modules/
# Builds
dist/
build/
.vite/
# Environment and secrets
.env
.env.*
!.env.example
# Local runtime data
data/
tmp/
temp/
logs/
*.log
# Local infrastructure volumes
.postgres/
.minio/
# OS / editor
.DS_Store
Thumbs.db
.idea/
.vscode/

View File

@ -0,0 +1,58 @@
# seo_mode Workspace
Рабочая папка для разработки seo_mode.
Архитектурная документация лежит в `SEO-farm/`.
## Текущий статус
Код приложения уже инициализирован.
Готов текущий MVP-срез:
- React/Vite UI и Node.js backend.
- Postgres + MinIO через `infra/docker-compose.yml`.
- Миграции базовой проектной модели.
- Создание, переименование, копирование и удаление проектов.
- Импорт папки через browser folder picker / drag-and-drop в MinIO.
- Скан проекта, `scan_version`, stable `pages/media`, повторный scan.
- `project-index.json` как JSON-снимок скана в MinIO.
- Baseline audit по выбранным страницам: title, description, H1/headings, видимый текст, media/ALT/filename.
- UI выбора страниц деревом по путям, severity-фильтры, media issue list и просмотр ошибок только по выбранным страницам.
- Page Workspace MVP: preview страницы, ошибки выбранной страницы, логические секции, section editor, save/reset без изменения исходного проекта.
Текущий этап: довести `Page Workspace / Draft Model` до кликабельного редакторского сценария и подготовить следующий шаг `Model Provider / Codex Exec`.
## Базовые решения
- `Postgres` — основной источник истины.
- `MinIO/S3` — хранилище raw artifacts, snapshots, reports, backups и exports.
- `browser_folder` — текущий рабочий source adapter для импорта папки в MinIO и будущей hosted-версии.
- `local_folder` — dev/local adapter для ручного абсолютного пути.
- `scan_version + reconcile` — модель повторного скана без потери утверждённых решений.
- `Apply` проходит через changeset, dry-run, backup и rollback.
## Структура
```text
app/ React/Vite UI
server/ Node.js backend
infra/ Postgres + MinIO local infrastructure
SEO-farm/ архитектурная документация
```
## Запуск
```powershell
npm install
Copy-Item .env.example .env
docker compose -f infra/docker-compose.yml up -d
npm run db:migrate
npm run dev
```
Если Docker Desktop не установлен, backend можно запустить без deep health checks:
```powershell
npm run dev:server
```

View File

@ -0,0 +1,636 @@
# checkOverseo Specification
## 1. Цель
`checkOverseo` — локальный инструмент проверки перетошноты, заспамленности, водности и ключевых повторов в SEO-правках.
Это не платная интеграция и не внешний API. Мы используем Text.ru только как референс по смыслу показателей и устройству SEO-анализа текста.
Главный вопрос инструмента:
```text
Текст нормально оптимизирован или уже переблеван ключами?
```
## 2. Референс: что берём из Text.ru как идею
Из Text.ru SEO-анализатора берём не API, а логику показателей:
- количество символов;
- количество слов;
- процент водности;
- процент заспамленности;
- список ключей;
- группы ключей;
- смешанные слова из разных алфавитов;
- подсветка проблемных повторов, если получится реализовать.
Источники для ориентира:
- Text.ru API/check description: https://text.ru/api-check
- Text.ru SEO-анализ: https://text.ru/seo
- Text.ru FAQ SEO-анализ: https://text.ru/faq/instrukcii/seo-analiz-teksta
Важно: Text.ru — только ориентир. Мы не подключаем их API и не платим за него.
## 2.1. Разбор публичной модели Text.ru
По публичному описанию Text.ru SEO-анализатор устроен как набор параллельных проверок текста:
```text
raw text
|
+-- character/word counter
+-- keyword extractor
+-- keyword group builder
+-- water detector
+-- spam detector
+-- mixed alphabet detector
+-- highlighter / word position mapper
```
### 2.1.1. Базовые счётчики
Text.ru считает:
```text
count_chars_with_space
count_chars_without_space
count_words
```
Нам нужно повторить это локально на каждом scope:
```text
page
section
title
description
heading
alt
filename
```
### 2.1.2. Ключи
Text.ru описывает поиск поисковых ключей в тексте, количество вхождений и морфологические варианты. В API manual это выражено как:
```text
list_keys:
key_title
count
```
Где `count` — количество вхождений ключа во всех формах.
Наша локальная версия:
```text
known_keys:
- берём из keywords.yml
- exact phrase
- soft_forms
detected_keys:
- считаем частотные unigram/bigram/trigram
- отбрасываем служебный мусор правилами
- показываем как "detected", не как утверждённый SEO-ключ
```
Важно: мы не обязаны идеально повторять морфологию Text.ru. Для MVP достаточно:
```text
exact phrase
+ manually/model-defined soft_forms
+ detected repeated phrases
```
### 2.1.3. Группы ключей
Text.ru группирует ключи по составу слов и сортирует по числу значимых слов. В API manual это:
```text
list_keys_group:
key_title
count
sub_keys:
key_title
count
```
Пример идеи:
```text
"check text" -> group
"check"
"text"
```
Наша локальная версия:
```text
keyword group:
parent phrase:
- full phrase
- single-word subkeys
- soft_forms
- repeated close phrases
```
Зачем это нужно:
```text
- видеть, что текст не только повторяет один точный ключ;
- видеть, что одна группа терминов забила секцию;
- ловить ситуацию, когда разные формы одного смысла всё равно создают перетошноту.
```
### 2.1.4. Водность
Text.ru определяет воду как процент стоп-слов, фразеологизмов, оборотов, соединительных слов и фраз, которые не несут смысловой нагрузки.
Публичные пороги:
```text
до 15% естественное содержание воды
15-30% превышенное содержание воды
от 30% высокое содержание воды
```
Наша локальная версия не должна быть обычным stopword-remover. Нам нужен редакторский сигнал:
```text
water_score =
служебные слова
+ слабые бизнес-фразы
+ общие SEO-обороты
+ абстрактные фразы без объекта
+ предложения без конкретного действия/сущности
```
То есть мы считаем приблизительный процент, но в отчёте показываем не только число, а конкретные причины.
### 2.1.5. Заспамленность
Text.ru описывает заспамленность как количество поисковых ключевых слов в тексте.
Публичные пороги:
```text
до 30% естественное содержание ключевых слов
30-60% оптимизированный текст
от 60% сильно оптимизированный / заспамленный текст
```
Наша локальная версия считает это по двум слоям:
```text
1. known-key spam:
- primary keywords
- secondary keywords
- soft_forms
2. detected-key spam:
- частотные слова
- частотные биграммы
- частотные триграммы
- группы похожих ключей
```
Для нас важнее не общий процент по странице, а риск по секции:
```text
section spam > page spam
```
Потому что лендинг может быть нормальным целиком, но отдельная карточка будет переблевана ключом.
### 2.1.6. Mixed words
Text.ru ищет слова, где смешаны символы разных алфавитов со схожим написанием.
Наша локальная версия:
```text
mixedWordsDetector:
- кириллица + латиница внутри одного слова
- похожие символы: а/a, о/o, е/e, р/p, с/c, х/x
- выдавать позиции и snippets
```
Это не SEO-ключи, но полезная техническая проверка качества текста.
### 2.1.7. Подсветка и позиции
В Text.ru API есть режимы, где возвращаются:
```text
words
text_view
words_pos
```
Это значит, что их интерфейс может подсвечивать проблемные слова/ключи по позициям.
Наша локальная версия должна с самого начала хранить offsets:
```ts
type Token = {
text: string;
normalized: string;
start: number;
end: number;
sentenceIndex: number;
paragraphIndex: number;
};
```
Зачем:
```text
- подсветить ключ в UI;
- показать proximity;
- показать повтор рядом;
- показать water phrase;
- показать mixed word;
- дать Codex точный snippet для правки.
```
## 3. Архитектура
```text
checkOverseo
|
+-- TextNormalizer
| +-- normalize spaces
| +-- normalize quotes/dashes where needed
| +-- lowercase for counting
| +-- keep original text for report snippets
|
+-- Tokenizer
| +-- words
| +-- sentences
| +-- bigrams
| +-- trigrams
|
+-- KeywordCounter
| +-- exact keywords
| +-- soft_forms from keywords.yml
| +-- key groups
|
+-- NauseaAnalyzer
| +-- classic nausea
| +-- academic nausea
| +-- phrase repetition
|
+-- WaterAnalyzer
| +-- local water-word dictionary
| +-- weak phrase markers
| +-- business-cliche markers
|
+-- ProximityAnalyzer
| +-- same keyword too close
| +-- repeated key in neighboring sentences
|
+-- ScopeAnalyzer
| +-- page
| +-- section
| +-- title
| +-- description
| +-- h1/h2
| +-- alt
| +-- filename
|
+-- ReportBuilder
+-- warnings
+-- scores
+-- snippets
+-- verdict
```
## 4. Вход
```ts
type CheckOverseoInput = {
text: string;
originalText?: string;
scope: "page" | "section" | "title" | "description" | "heading" | "alt" | "filename";
sourceId: string;
sectionId?: string;
keywords: KeywordEntry[];
pageMap: PageMap;
options?: CheckOverseoOptions;
};
```
## 5. Источник правил
### 5.1. `keywords.yml`
```yaml
keywords:
- phrase: "операционная модель бизнеса"
type: "primary"
section: "hero"
exact_limit: 1
max_section_count: 2
soft_forms:
- "операционная модель"
- "модель бизнеса"
status: "use"
```
### 5.2. `page-map.yml`
```yaml
page:
primary_intent: "главный смысл страницы"
forbidden_intents: []
sections:
hero:
meaning: "смысл секции"
primary_keywords: []
secondary_keywords: []
max_lines_hint: 6
```
### 5.3. Local dictionaries
```text
data/rules/
water-words-ru.yml
weak-business-phrases.yml
negative-keyword-intents.yml
safe-technical-terms.yml
```
Словари нужны не для автоматического переписывания, а для предупреждений.
## 6. Проверки
### 6.1. Counts
```text
- chars_with_space
- chars_without_space
- words_count
- sentences_count
- average_sentence_length
```
### 6.2. Keyword counts
```text
- exact keyword count
- soft_forms count
- primary keyword count
- secondary keyword count
- key group count
- keyword density by page
- keyword density by section
```
### 6.3. Classic nausea
Классическая тошнота считается как корень из максимальной частоты самого повторяемого слова или ключевой фразы.
```text
classic_nausea = sqrt(max_term_frequency)
```
Используется как локальный индикатор частого повторения.
### 6.4. Academic nausea
Академическая тошнота считается как доля самого частого слова/термина в общем количестве слов.
```text
academic_nausea_percent = max_term_frequency / words_count * 100
```
Также считаем отдельно:
```text
keyword_academic_nausea_percent
```
Чтобы видеть не просто частое служебное слово, а именно перебор нужного нам ключа.
### 6.5. Phrase repetition
```text
- repeated exact phrase
- repeated bigram
- repeated trigram
- same sentence starts
- repeated sentence skeleton
```
### 6.6. Proximity
```text
- один и тот же ключ повторяется в соседних предложениях;
- один и тот же ключ повторяется в соседних абзацах;
- primary keyword стоит слишком близко к прошлому вхождению;
- alt повторяет heading без причины;
- filename повторяет alt и ключ без смысла.
```
### 6.7. Water score
Локальная водность — не попытка идеально повторить Text.ru. Это наш сигнал:
```text
water_score = water_words_count / words_count * 100
```
Плюс отдельные предупреждения:
```text
- слабые бизнес-фразы;
- общие SEO-фразы;
- много абстрактных существительных;
- длинные предложения без конкретного объекта;
- фразы, которые не добавляют смысла.
```
### 6.8. Scope rules
Разные зоны проверяются по-разному:
```text
page:
- общий риск заспамленности
- общий список ключей
section:
- соответствие ключей секции
- локальный перебор
- конфликт с meaning секции
title/description:
- нет ли повторения одного ключа дважды
- нет ли набора ключей вместо нормального сниппета
heading:
- нет ли keyword stuffing
- заголовок соответствует секции
alt:
- описывает изображение
- ключ встроен естественно
- нет списка ключей
filename:
- имя файла осмысленное
- нет набора ключей через дефисы
- имя файла не конфликтует с содержанием изображения
```
## 7. Пороговые значения MVP
Пороги должны быть настраиваемыми, потому что B2B/технические тексты неизбежно повторяют термины.
```yaml
thresholds:
keyword_density:
section_warning: 3
section_problem: 5
exact_keyword:
default_limit: 2
classic_nausea:
warning: 4
problem: 7
academic_nausea_percent:
warning: 7
problem: 10
proximity:
min_sentences_between_same_keyword: 2
water_percent:
warning: 20
problem: 30
```
Пороги не должны автоматически запрещать текст. Они создают предупреждения.
## 8. Report
```ts
type OverSeoReport = {
scope: "page" | "section" | "title" | "description" | "heading" | "alt" | "filename";
sourceId: string;
sectionId?: string;
counts: {
charsWithSpace: number;
charsWithoutSpace: number;
words: number;
sentences: number;
};
scores: {
waterPercent?: number;
classicNausea?: number;
academicNauseaPercent?: number;
maxKeywordDensityPercent?: number;
};
keys: Array<{
phrase: string;
count: number;
densityPercent: number;
type?: "primary" | "secondary" | "soft_form" | "detected";
sectionAllowed?: boolean;
}>;
keyGroups: Array<{
phrase: string;
count: number;
subKeys: Array<{
phrase: string;
count: number;
}>;
}>;
warnings: Array<{
severity: "info" | "warning" | "critical";
code: string;
message: string;
phrase?: string;
count?: number;
limit?: number;
snippet?: string;
}>;
verdict: {
status: "ok" | "review" | "problem";
summary: string;
};
};
```
## 9. UI
UI должен показывать:
```text
Summary:
- status
- words count
- water score
- classic nausea
- academic nausea
- max keyword density
Keywords:
- phrase
- count
- density
- allowed section
- warnings
Warnings:
- severity
- reason
- snippet
- suggested action
Scopes:
- page
- sections
- title/description
- headings
- alt
- filename
```
## 10. Codex role
Codex не считает метрики вместо инструмента.
Codex используется после отчёта:
```text
1. Объяснить предупреждения человеческим языком.
2. Предложить варианты исправления.
3. Сохранить смысл и тон.
4. Не увеличивать текст без необходимости.
5. Не вставлять ключи, если отчёт показывает перебор.
```
## 11. Главное ограничение
`checkOverseo` не должен становиться тупой машиной запретов.
Он показывает риски:
```text
перебор
слишком близко
слишком часто
слишком водно
слишком похоже на набор ключей
```
Финальное решение принимает пользователь.

View File

@ -0,0 +1,36 @@
# seo_mode design system notes
Короткая памятка, чтобы интерфейс не расползался при быстрых итерациях.
## Визуальный тон
- База: светлая Apple-like система, много воздуха, мягкие панели, скругления и спокойный серый фон.
- Главный акцент: тёмный градиент `#1f1f21 -> #3a332e`, как у активного этапа и основных submit/action-кнопок.
- Вторичный акцент: оранжевый `--accent: #f27a1a`, только для выбора папки, drag-and-drop и редких импортных действий.
- Избегаем случайной зелени/голубого/фиолета, если это не специально введённый статус.
## Кнопки
- Primary actions: `Создать проект`, `Анализировать`, будущие основные шаги этапов. Используют тёмный градиент активного этапа.
- Import action: `Выбрать папку`. Остаётся оранжевой, потому что это отдельное действие загрузки/импорта.
- Icon actions: копировать, удалить, настройки. Круглые `42x42`, нейтральная светлая поверхность, тонкая граница, мягкая тень.
- Danger icon: нейтральная круглая поверхность в покое, красный значок; красная заливка только на hover.
## Типографика
- Заголовки секций: плотные, тёмные, без лишних декоративных цветов.
- Helper copy под инпутами: `13px`, `font-weight: 400`, приглушённый `--muted`, line-height около `1.38`.
- Placeholder: всегда normal/400, приглушённый, без жирности.
- Иконки рядом с заголовками секций выравниваем по первой строке заголовка, а не по центру всего блока с описанием.
- Не смешивать в одной карточке слишком много жирных уровней: жирный только у заголовка и главного действия.
## Карточки и сетки
- Основная ширина страницы: не заужать без причины, держать рабочую ширину около `min(1180px, 100%)`.
- Карточки: светлая полупрозрачная поверхность, `var(--radius-lg/xl)`, тонкая линия `var(--line)`, мягкая тень только на крупных контейнерах.
- Не вкладывать много одинаковых карточек друг в друга без визуальной причины.
- Каждый этап сверху должен быть отдельной логической страницей, а предыдущие настройки на следующих этапах показываем компактно.
## Рабочее правило
Перед новой UI-правкой сначала проверяем, нет ли уже подходящего класса/паттерна. Если стиль повторяется второй раз, выносим его в общий паттерн, а не лепим ещё один локальный костыль.

View File

@ -0,0 +1,459 @@
# SEO Farm Implementation Guardrails
## 1. Зачем этот документ
Этот файл фиксирует порядок разработки и технические ограничения, чтобы user flow, backend pipeline и инструменты не начали спорить друг с другом.
Главная идея:
```text
UI показывает понятный SEO-процесс.
Backend внутри выполняет строгую техническую последовательность.
Инструменты не смешивают зоны ответственности.
Target project меняется только после ручного Apply.
```
## 2. User Flow vs Backend Pipeline
Для пользователя первый смысловой шаг после создания проекта — “Проанализировать проект”.
В UI это один сценарий:
```text
Create Project
-> Analyze Project
-> Baseline Audit Result
```
Но backend не может делать audit без технической подготовки. Внутренний порядок должен быть таким:
```text
1. validate targetProjectPath
2. scan filesystem
3. filter ignored folders/files
4. detect candidate files
5. parse HTML/MD/text/CSS references
6. build projectIndex
7. map pages
8. map sections
9. map media assets
10. create source mapping
11. run baseline audit
12. save scan + audit result
13. return UI-ready report
```
Для будущих источников backend должен мыслить не только `targetProjectPath`, а `project source`:
```text
MVP:
local_folder -> targetProjectPath
Later:
git_repo
zip_upload
site_crawl
```
Нельзя делать:
```text
Baseline Audit
-> потом parse
```
Правильно:
```text
Scan / Parse / Map
-> Baseline Audit
```
В UI слово “parse” лучше не выносить отдельным этапом. Это внутренняя операция.
## 3. Строгий MVP Pipeline
```text
Project Setup
input: project name, project source
output: project record
Project Scan
input: project source
output: projectIndex
Baseline Audit
input: projectIndex
output: audit report, page/media/source issues
Page Workspace
input: selected page/group
output: originalText, workingText, sectionMap
Meaning Extraction
input: originalText, sectionMap, meta/headings/media context
output: section meanings, seed queries, competing intents
Seed Review
input: proposed seeds
output: approved seeds
Wordstat Collection
input: approved seeds
output: raw Wordstat results, ВЧ/СЧ/НЧ groups
Keyword Cleaning
input: raw Wordstat results
output: approved keyword decisions
Keyword Map
input: approved keywords, section meanings, current text
output: approved keyword-to-section map
Optimization Plan
input: keyword map, audit, current workingText, design limits
output: approved edit plan
Rewrite Workspace
input: approved edit plan
output: edited workingText
Media SEO
input: media map, section context, keyword map
output: approved alt/title/aria/caption/filename suggestions
Final Validation
input: workingText, keyword map, media map, approved media changes
output: validation report
Apply / Export
input: approved text/media changes
output: changeset, updated target project, reports/export
```
## 4. Tool Boundaries
### 4.1. `checkOverseo`
Отвечает за SEO-перебор:
```text
- точные вхождения;
- soft forms;
- перетошнота;
- заспамленность;
- повторы рядом;
- повторяющиеся биграммы/триграммы;
- перебор ключа в title/description/h1/h2;
- keyword stuffing в alt/filename;
- конфликт ключа со смыслом секции.
```
Не отвечает за:
```text
- стиль бренда;
- красивость текста;
- смысловую редактуру;
- выбор ключей из Wordstat.
```
### 4.2. `ru-text`
Отвечает за редакторский фильтр после SEO-правок:
```text
- тон;
- читаемость;
- канцелярит;
- мутные формулировки;
- редакторские и типографические проблемы.
```
Не отвечает за:
```text
- сбор ключей;
- частотность;
- Wordstat;
- карту ключей;
- SEO-плотность.
```
### 4.3. `keywordService`
Отвечает за чистку Wordstat-результатов:
```text
- нормализация запросов;
- дедупликация;
- project blacklist;
- мусорные интенты;
- группировка близких запросов;
- Codex-классификация спорных запросов;
- user approval.
```
Это не `stopword`-инструмент и не удаление слов из текста.
### 4.4. Wordstat MCP
Wordstat MCP — только “рука в Wordstat”.
Он:
```text
- получает seed;
- возвращает запросы и частотность;
- сохраняет raw data.
```
Он не:
```text
- решает, что использовать;
- пишет текст;
- строит карту ключей;
- проверяет переспам.
```
### 4.5. Codex Exec
Codex через `codex exec` используется для смысловых задач:
```text
- выделить темы;
- предложить seed queries;
- классифицировать спорные ключи;
- предложить keyword map;
- сформировать optimization plan;
- предложить варианты правки;
- предложить alt/caption/filename.
```
Codex не должен напрямую менять target project до `Apply`.
## 5. Apply Safety Rules
До ручного Apply target project считается read-only.
Разрешённые изменения после approval:
```text
- visible text;
- title;
- description;
- headings;
- alt/title/aria;
- captions;
- approved filenames.
```
Запрещено без отдельного подтверждения:
```text
- менять layout;
- менять CSS/JS-логику;
- рефакторить код сайта;
- удалять файлы;
- менять структуру приложения SEO Farm;
- публиковать на хостинг.
```
Перед Apply UI обязан показать:
```text
- affected files;
- text diff;
- media diff;
- filename rename plan;
- validation status;
- changeset summary.
```
Apply должен идти транзакционно:
```text
1. build changeset
2. run dry-run
3. backup affected files to object storage
4. apply text/media/filename changes through backend
5. verify references and rename results
6. rollback if any critical step fails
```
## 5.1. Repeated Scan / Reconcile Rules
Повторный scan не должен перетирать уже принятые решения.
Правильно:
```text
new scan
-> create scan_version
-> create page/section/media versions
-> run reconcile
-> preserve stable logical IDs
-> show review summary to user
```
Стабильные идентификаторы:
```text
page_id
section_id
media_id
```
Текущий `DOM selector` — это версия привязки, а не identity.
Статусы reconcile:
```text
matched
changed
new
orphaned
conflicted
```
Решения пользователя должны ссылаться на logical IDs:
```text
- keyword decisions
- drafts
- media approvals
- changesets
```
## 6. Storage Risks
Нельзя хранить всё только в памяти.
Минимально нужно:
```text
Postgres:
- projects
- project_sources
- scan_versions
- pages / page_versions
- sections / section_versions
- media_assets / media_versions
- runs
- seeds
- wordstat_jobs / wordstat_results
- keyword_decisions
- keyword_map_items
- drafts / draft_items
- validations / validation_issues
- changesets / changeset_items
Object Storage:
- raw Wordstat dumps
- originalText snapshots
- workingText drafts
- reports
- previews / video frames
- backups
- exports
```
Правило:
```text
Postgres = source of truth
Object Storage = heavy artifacts
YAML/JSON/Markdown = export/debug only
```
Риск:
```text
Если workingText не отделить от originalText,
модельная ошибка может испортить исходник без понятного отката.
```
MVP может хранить одну активную рабочую версию на секцию, но scan versions, snapshots и changesets должны быть полноценными уже с первого релиза.
## 7. Development Risk Checklist
### 7.1. Pipeline risks
- Audit запущен до parse/map.
- UI показывает parse как отдельную пользовательскую задачу.
- Project scan требует ручного выбора файлов.
- Многостраничный сайт превращается в ручной список `index.html`.
- Повторный scan затирает уже утверждённые решения без предупреждения.
- Решения по секции привязаны только к текущему selector и теряются после перестройки DOM.
### 7.2. Model risks
- Codex меняет target project до Apply.
- Prompt не ограничивает зону правок.
- Model output невалидный JSON, а backend применяет его без проверки.
- Модель смешивает SEO-правки с редизайном или рефакторингом кода.
### 7.3. Keyword risks
- Wordstat results считаются готовой keyword map.
- Keyword Cleaning превращается в ручную таблицу без assisted classification.
- ВЧ-запросы насильно пихаются в каждую секцию.
- `article`-ключи запихиваются в главную страницу вместо будущих материалов.
- Нет user approval перед использованием ключей.
### 7.4. Text quality risks
- `checkOverseo` пытается оценивать тон вместо переспама.
- `ru-text` используется как SEO-решатель вместо редакторского фильтра.
- Проверка длины игнорирует дизайн-блоки.
- Правки раздувают текст и ломают визуальную секцию.
### 7.5. Media SEO risks
- Alt набивается ключами без связи с изображением.
- Decorative media получают SEO-alt.
- Видео обрабатывается как `<img alt="">`, хотя у video нет обычного alt.
- Filename меняется без обновления ссылок в HTML/CSS/srcset.
- Rename plan применяется без diff.
### 7.6. Apply/export risks
- Apply происходит без changeset.
- Apply происходит без dry-run и backup.
- Rename/update references падает посередине без rollback.
- Export не включает отчёты.
- После apply невозможно понять, какие SEO-решения были приняты.
- Автопубликация появляется раньше ручного контроля.
## 8. Recommended Development Order
Чтобы не собрать хаос, реализация должна идти так:
```text
1. Project storage + Postgres + Object Storage
2. Project setup screen
3. project source validation
4. project scan
5. scan_versions + reconcile
6. projectIndex
7. baseline audit
8. page workspace with originalText/workingText
9. CodexExecProvider
10. meaning extraction
11. seed review
12. Wordstat MCP integration
13. keywordService
14. keyword map
15. optimization plan
16. rewrite workspace
17. media SEO
18. checkOverseo
19. ru-text
20. final validation report
21. diff + changeset
22. apply with dry-run/backup/rollback
23. export/history
```
Нельзя начинать с rewrite UI до появления `originalText/workingText`, иначе легко получить приложение, которое красиво пишет текст, но не умеет безопасно применять изменения.

View File

@ -0,0 +1,44 @@
# SEO Farm
Локальный SEO-редакторский контур для работы с существующими сайтами: scan проекта, baseline audit, Wordstat, keyword map, rewrite workspace, media SEO, final validation и ручной Apply через changeset.
## Статус
Архитектурная документация остаётся источником направления, но код приложения уже инициализирован.
Готов текущий MVP-срез:
- базовый React/Vite UI;
- Node.js backend;
- Postgres как source of truth;
- MinIO/S3-compatible storage для артефактов;
- создание, переименование, копирование и удаление проектов;
- импорт папки сайта через browser folder picker / drag-and-drop;
- scan project + `project-index.json`;
- stable pages/media records и repeated scan;
- baseline audit;
- выбор страниц для дальнейшей работы деревом по путям;
- severity-фильтры и отдельный media issue list;
- отображение ошибок только по выбранным страницам;
- Page Workspace MVP: preview страницы, ошибки выбранной страницы, логические секции, section editor, save/reset без изменения исходного проекта.
Текущий рабочий этап: довести `Page Workspace / Draft Model` до кликабельного редакторского сценария.
## Основные документы
- `SEO_FARM_BRIEF.md` — общее ТЗ, цели продукта и MVP scope.
- `SEO_FARM_ARCHITECTURE.md` — архитектура, storage, backend/frontend слои.
- `USER_FLOW.md` — пользовательский сценарий.
- `IMPLEMENTATION_GUARDRAILS.md` — pipeline, границы инструментов и риски.
- `UI_UX_SPEC.md` — UI/UX экраны, состояния и safety-паттерны.
- `TASK_MANAGER.md` — поэтапный план реализации.
- `CHECK_OVERSEO_SPEC.md` — спецификация локальной проверки переспама.
## Зафиксированные архитектурные решения
- `Postgres` — основной источник истины.
- `MinIO/S3` — хранилище raw artifacts, snapshots, reports, backups и exports.
- `browser_folder` — текущий рабочий adapter для импорта папки в MinIO и будущего hosted-сценария.
- `local_folder` — dev/local adapter для ручного абсолютного пути.
- `scan_version + reconcile` — модель повторного скана без потери утверждённых решений.
- `Apply` проходит через changeset, dry-run, backup и rollback.

View File

@ -0,0 +1,833 @@
# SEO Farm Architecture
## 1. Общая схема
```text
React/Vite UI
|
v
Local Backend
|
+-- Project Storage
| |
| +-- Postgres DB (source of truth)
| +-- Object Storage
| | +-- MinIO in local/dev
| | +-- S3 in hosted mode later
| +-- Project Source Adapters
| | +-- local_folder (MVP)
| | +-- later: git_repo
| | +-- later: zip_upload
| | +-- later: site_crawl
| +-- project_sources
| +-- scan_versions
| +-- page/section/media logical records
| +-- page_versions / section_versions / media_versions
| +-- originalText snapshots
| +-- workingText drafts
| +-- run history
| +-- changesets
| +-- backups
| +-- exports
| +-- reports/raw dumps/previews/frames
|
+-- Task Runner
|
+-- Model Provider
| |
| +-- MVP: Codex Exec Provider
| | |
| | +-- codex exec
| |
| +-- Later: OpenAI API Provider
| +-- Later: Other API Provider
| +-- Later: Local LLM Provider
|
+-- SEO Tools
| |
| +-- HTML/Text Extractor
| +-- Structure Checker
| +-- Keyword Checker
| +-- Over-SEO Checker
| +-- Page Map Checker
|
+-- Wordstat Layer
| |
| +-- MCP-KV Wordstat
|
+-- Text Quality Layer
| |
| +-- ru-text
|
+-- Media SEO Layer
|
+-- HTML media parser
+-- optional: image-size
+-- optional: Sharp
+-- optional: FFmpeg
+-- Model Provider for image/video description
```
## 1.1. Статус инструментов
```text
MVP, точно используем:
- React
- Vite
- Node.js backend
- codex exec
- ModelProvider abstraction
- MCP-KV Wordstat
- ru-text
- Cheerio
- local SEO checks
- Postgres
- MinIO or other S3-compatible object storage
- structured JSON model outputs
- YAML/JSON/Markdown exports for debug/import-export only
Опционально для Media SEO, подключать когда дойдём до медиа:
- image-size
- Sharp
- FFmpeg
Не используем в MVP:
- автопостинг
- Яндекс Метрика
- robots/sitemap editor
- hosted/cloud mode
- Codex skills as dependency
```
Future enhancement:
```text
Codex skills can be added later to improve repeatable model workflows:
- seo-seed-extractor
- seo-keyword-classifier
- seo-copy-editor-ru
- media-seo-alt-writer
Skills are not required for MVP. First stabilize tools and reports.
```
## 1.2. Project Source Adapters
```text
SourceAdapter
|
+-- local_folder
| +-- MVP source type
| +-- uses targetProjectPath
|
+-- git_repo
| +-- future hosted/local source
|
+-- zip_upload
| +-- future hosted/local source
|
+-- site_crawl
+-- future audit-only source
```
For MVP we start with `local_folder`, but the application should keep the source type in the database from day one so local and hosted modes use the same upper pipeline.
## 2. Основной поток данных
```text
1. User creates/selects SEO Farm project
|
v
2. User sets project source for the tested site/codebase
|
+-- MVP: local_folder -> targetProjectPath
|
v
3. Backend creates scan_version, scans source, and builds projectIndex
|
v
4. Baseline Audit runs on discovered pages/assets/media
|
v
5. User selects page or page group for SEO work
|
v
6. Backend creates originalText snapshots and workingText drafts
|
v
7. Model Provider extracts meanings and seed queries
|
v
8. User reviews/edits seed queries
|
v
9. Wordstat Layer collects keyword data
|
v
10. keywordService cleans/classifies keyword results
|
v
11. User reviews keyword decisions
|
v
12. Keyword Map Builder proposes section mapping and limits
|
v
13. User approves/edits Keyword Map
|
v
14. Optimization Plan is generated and approved
|
v
15. Model Provider proposes text edits into workingText
|
v
16. Media SEO proposes alt/caption/filename edits
|
v
17. SEO Tools + ru-text validate text and media edits
|
v
18. UI shows report and diff
|
v
19. User applies changes to target project
|
v
20. User can export archive/reports
```
MVP works with existing code/text only:
```text
existing HTML / existing draft / existing site
-> analyze
-> collect keywords
-> edit
-> final validate
```
Creating text from scratch is a future mode, not part of MVP.
## 3. UI Modules
```text
app/
Projects
-> create/select local project
-> set target project path
-> scan project
Project Dashboard
-> project status
-> discovered pages
-> baseline audit status
-> recent runs
-> working drafts
-> changesets
-> exports
Baseline Audit
-> project/page structure
-> title/description/h1/h2
-> discovered text blocks
-> media issues
-> technical SEO issues
Pages / Source
-> discovered pages
-> selected page
-> extracted sections
-> extracted headings
-> original text
-> working text
Seeds
-> seed queries from source text
-> manual seed editing
-> select seeds for Wordstat
Wordstat
-> run MCP-KV Wordstat
-> show frequency/dynamics/regions
-> raw keyword results
-> high/mid/low groups
Keyword Cleaning
-> normalize/deduplicate Wordstat queries
-> mark obvious trash
-> classify ambiguous queries through Codex
-> approve/edit statuses: use/support/article/risky/trash
Keyword Map
-> suggested keyword-to-section map
-> approve/edit suggestions
-> primary/secondary keywords
-> soft forms
-> exact-match limits
Optimization Plan
-> sections to edit
-> sections to leave unchanged
-> keywords to add/reduce
-> media tasks
-> length/tone risks
Rewrite
-> select section
-> edit working text
-> generate variants through Model Provider
-> run SEO checks
-> run ru-text
-> show original vs working diff
-> apply only after user approval
Media SEO
-> list images/videos/posters
-> current alt/title/aria/captions
-> preview
-> suggested alt/descriptions
-> media SEO warnings
Final Validation
-> title/description/h1/h2
-> competing meanings
-> keyword usage
-> over-SEO warnings
-> text length/design limits
-> ru-text status
-> media SEO status
Apply / Export
-> affected files
-> changeset
-> manual approval
-> apply to target project
-> zip target project
-> export reports
History / Reports
-> scans
-> audits
-> Wordstat runs
-> validation reports
-> media SEO reports
-> keyword decisions
-> rewrites
-> changesets
-> exports
```
## 4. Backend Modules
```text
server/
routes/
projects
scan
pages
sources
seeds
wordstat
keywords
keyword-cleaning
keyword-map
optimization-plan
audit
rewrite
media
validate
apply
export
history
reports
providers/model/
ModelProvider.ts
CodexExecProvider.ts
OpenAiApiProvider.ts # future
CustomHttpProvider.ts # future
providers/storage/
StorageProvider.ts
MinioStorageProvider.ts
services/
db
projectSourceService
storageProvider
taskRunner
projectStorage
projectScanService
reconcileService
pageWorkspaceService
wordstatService
keywordService
keywordMapService
optimizationPlanService
seoAuditService
ruTextService
mediaAuditService
validationService
changesetService
applyService
exportService
reportService
```
## 5. Tool Responsibilities
| Layer | Tool | Responsibility |
|---|---|---|
| UI | React | Interface, state, workflows |
| UI | Vite | Dev server and build |
| Backend | Node.js | Local server and task orchestration |
| Backend | Express/Fastify | HTTP API for UI |
| Storage | Postgres | Source of truth for projects, sources, versions, approvals, runs, validations, changesets |
| Storage | MinIO/S3 | Raw dumps, previews, reports, exports, backups, scan snapshots |
| Model | `codex exec` | MVP model execution |
| Model | ModelProvider | Switch between Codex/API/local models |
| Wordstat | MCP-KV Wordstat | Raw keyword data from Yandex Wordstat |
| Text parsing | Cheerio | Parse HTML and extract title, meta, headings, text blocks, media tags |
| SEO checks | Local scripts | Title, description, h1/h2, keywords, spam, length |
| Text quality | ru-text | Editorial SEO correction, tone, readability, typographic checks |
| Media parsing | Cheerio | Find img/picture/video/source/poster |
| Images | image-size, optional | Read image dimensions for media report |
| Images | Sharp, optional | Generate previews/thumbnails if browser preview is not enough |
| Video | FFmpeg, optional | Extract representative frames from videos |
| Files | YAML/JSON/Markdown | Export, debug, import/export artifacts only; not the main storage layer |
## 6. Model Provider Abstraction
```text
ModelProvider
|
+-- CodexExecProvider
| +-- calls: codex exec
| +-- use case: local MVP
|
+-- OpenAiApiProvider
| +-- calls: OpenAI API
| +-- use case: remote app / hosted version
|
+-- CustomHttpProvider
| +-- calls: any compatible API
| +-- use case: future integrations
|
+-- LocalLlmProvider
+-- calls: local model runtime
+-- use case: offline/private mode
```
## 7. Wordstat Flow
```text
Seed queries
|
v
wordstatService
|
v
MCP-KV Wordstat
|
+-- top requests
+-- related requests
+-- dynamics
+-- regions
|
v
raw keyword dump
|
v
keywordService
|
+-- clean by local rules
+-- classify
+-- frequency type: high/mid/low
+-- status: use/support/article/risky/trash
```
## 7.1. Keyword Cleaning Logic
Чистка мусора — это не stopword/удаление частых слов. Это смысловая фильтрация Wordstat-ключей на релевантность проекту. Делает её `keywordService`: локальные правила проекта плюс Codex-классификация спорных запросов.
```text
Raw Wordstat keywords
|
v
keywordService
|
+-- normalize text
+-- remove duplicates
+-- apply negative intents
+-- apply project blacklist
+-- group close phrases
+-- mark obvious trash
+-- ask Model Provider to classify ambiguous phrases
|
v
User review
```
Базовые причины выкинуть запрос:
```text
- вакансии
- обучение / курсы
- скачать
- бесплатно
- реферат / диплом
- бытовой смысл вместо B2B
- чужая отрасль
- слишком широкий запрос без связи с продуктом
- запрос конфликтует с primary_intent страницы
```
Результат хранится как статус:
```text
use / support / article / risky / trash
```
## 8. Text SEO Flow
```text
Source text / HTML
|
v
extractText
|
+-- sections
+-- headings
+-- visible text
|
v
Model Provider
|
+-- meanings
+-- seed queries
|
v
SEO audit
|
+-- title/description/h1/h2
+-- competing meanings
+-- keyword usage
+-- over-SEO
+-- length/design limits
|
v
Rewrite variants
|
v
ru-text + SEO checks
|
v
User approval
```
## 8.1. Over-SEO Checker
Проверка перетошноты/переблева ключами описана отдельно в `CHECK_OVERSEO_SPEC.md`.
Text.ru используется только как референс по смыслу показателей. Их API не подключаем.
`seo-analyzer` и похожие HTML-аудиторы не заменяют этот слой: они полезны для title/h1/alt/структуры, но не решают именно перетошноту текста.
```text
checkOverseo
|
+-- exact keyword count
+-- soft_forms count from keywords.yml
+-- keyword density by section
+-- classic nausea: sqrt(max word/phrase frequency)
+-- academic nausea: top word/phrase share in text
+-- repeated exact phrases
+-- repeated bigrams/trigrams
+-- proximity: same keyword too close
+-- title/description/h1 overuse
+-- alt keyword stuffing
+-- section-level warnings
```
Источник правил:
```text
keywords.yml
- phrase
- soft_forms
- section
- exact_limit
- max_section_count
- status
- notes
page-map.yml
- primary_intent
- section meanings
- forbidden_intents
- max_lines_hint
```
В MVP не нужен сложный морфологический движок. Для русского текста сначала используем:
```text
- точные ключи;
- вручную/моделью заданные soft_forms;
- нормализацию регистра и пробелов;
- подсчёт повторов по секциям.
```
Если этого станет мало, позже можно добавить отдельный морфологический модуль, но он не входит в MVP.
## 9. Media SEO Flow
```text
Source HTML/CSS
|
v
extractMedia
|
+-- img
+-- picture/source/srcset
+-- video/source/poster
+-- CSS backgrounds
|
v
mediaAuditService
|
+-- current alt/title/aria
+-- filename
+-- captions
+-- section context
+-- dimensions
+-- file size
|
+-- optional image-size
+-- optional Sharp previews
+-- optional FFmpeg video frames
|
v
Model Provider
|
+-- describe image/video frame
+-- suggest alt/caption/description
|
v
SEO checks
|
+-- missing alt
+-- weak alt
+-- duplicate alt
+-- decorative media
+-- weak/non-descriptive filename
+-- keyword stuffing in alt
+-- missing video context
|
v
User approval
```
## 10. Storage Layout
```text
Postgres
- projects
- project_sources
- scan_versions
- pages
- page_versions
- sections
- section_versions
- media_assets
- media_versions
- runs
- seeds
- wordstat_jobs
- wordstat_results
- keyword_decisions
- keyword_map_items
- optimization_plans
- drafts
- draft_items
- validations
- validation_issues
- changesets
- changeset_items
- exports
Object Storage
- projects/project-id/scans/scan-version-id/project-index.json
- projects/project-id/raw/wordstat/*.json
- projects/project-id/snapshots/original/*.json
- projects/project-id/snapshots/working/*.json
- projects/project-id/reports/*.md
- projects/project-id/previews/*
- projects/project-id/video-frames/*
- projects/project-id/backups/changeset-id/*
- projects/project-id/exports/*
```
`project.json` can exist as export/debug metadata, but the source of truth lives in Postgres:
```json
{
"id": "nodedc",
"name": "Node.DC",
"source": {
"type": "local_folder",
"targetProjectPath": "C:/Users/const/Desktop/landing-correction"
},
"scan": {
"include": ["**/*.html", "**/*.md", "css/**/*.css", "images/**/*", "video/**/*"],
"exclude": ["node_modules/**", "dist/**", ".git/**"]
}
}
```
Postgres stores project state and history:
```text
projects
project_sources
scan_versions
pages
page_versions
sections
section_versions
media_assets
media_versions
runs
seeds
wordstat_jobs
wordstat_results
keyword_decisions
keyword_map_items
optimization_plans
drafts
draft_items
validations
validation_issues
changesets
changeset_items
exports
```
Rules:
```text
Postgres = source of truth
Object Storage = heavy/snapshot/report artifacts
YAML/JSON/Markdown = export/debug/import-export only
```
## 10.1. Repeated Scan / Reconcile Model
Every scan creates a new `scan_version`.
Important rule:
```text
DOM selector != entity identity
```
Stable identities:
```text
page_id
section_id
media_id
```
Versioned snapshots:
```text
page_version
section_version
media_version
```
When a new scan is created, backend runs reconcile:
```text
matched
changed
new
orphaned
conflicted
```
Matching should use more than one signal:
```text
- source path/url
- heading/context
- normalized text hash
- selector fingerprint
- neighboring structure
```
Approvals, keyword decisions, drafts, and media decisions must link to stable logical IDs, not only to the current selector.
Text editing model:
```text
target project file
|
v
extract text
|
+-- originalText snapshot
|
+-- workingText draft
|
+-- user edits
+-- Codex edits
+-- SEO checks
+-- ru-text checks
|
v
diff
|
v
user approval
|
v
changeset
|
+-- dry-run
+-- backup affected files to object storage
+-- apply through backend
+-- verify references/renames
+-- rollback on failure
|
v
apply to target project
```
MVP stores one `workingText` draft per source/section. Full draft versioning and rollback history are future enhancements.
## 11. External Links
Related local docs:
- `SEO_FARM_BRIEF.md`
- `USER_FLOW.md`
- `CHECK_OVERSEO_SPEC.md`
- `IMPLEMENTATION_GUARDRAILS.md`
- `UI_UX_SPEC.md`
- `TASK_MANAGER.md`
- Codex exec: https://developers.openai.com/codex/noninteractive
- Codex with ChatGPT plan: https://help.openai.com/en/articles/11369540-using-codex-with-your-chatgpt-plan
- React: https://react.dev/
- Vite: https://vite.dev/guide/
- MCP-KV Wordstat: https://mcp-kv.ru/docs/wordstat-mcp-setup
- MCP-KV SEO tools, optional/reference: https://mcp-kv.ru/mcp-server-seo
- ru-text: https://github.com/talkstream/ru-text
- Cheerio: https://cheerio.js.org/docs/api/
- image-size: https://github.com/image-size/image-size
- Sharp: https://www.npmjs.com/package/sharp
- FFmpeg: https://ffmpeg.org/ffmpeg-doc.html
- Text.ru SEO analysis reference: https://text.ru/seo
- Text.ru API/check description, reference for analysis model only: https://text.ru/api-check

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,642 @@
# SEO Farm User Flow
## 1. Главный принцип
SEO Farm работает как проектный сервис, а не как одноразовая форма.
```text
Пользователь создаёт проект
-> приложение сканирует сайт
-> делает baseline audit
-> пользователь выбирает страницы
-> система помогает собрать ключи
-> система предлагает карту ключей
-> пользователь правит тексты в working copy
-> checkOverseo ловит переспам
-> ru-text и редакторские проверки ловят плохой тон
-> пользователь применяет изменения
-> история работы сохраняется
```
MVP работает с уже существующим кодом/страницей/текстом. Создание текстов с нуля — отдельный будущий режим.
## 2. Project Lifecycle
### 2.1. Create Project
Пользователь создаёт проект и выбирает папку сайта:
```text
source.type = local_folder
targetProjectPath = C:/path/to/site
```
Пользователь не выбирает вручную `index.html`, `css`, `images` и другие файлы. Приложение само сканирует проект.
На этом шаге сохраняется:
```text
project
- id
- name
- source.type
- targetProjectPath
- createdAt
- updatedAt
- lastRunAt
```
### 2.2. Project Scan
Приложение сканирует папку проекта и строит `projectIndex`.
Для пользователя это часть одного действия “Проанализировать проект”. Внутри backend обязан соблюдать порядок:
```text
scan filesystem
-> parse candidate files
-> create scan_version
-> build projectIndex
-> map pages/sections/media
-> reconcile with previous scan_version
-> run baseline audit
-> show audit result in UI
```
Baseline audit не может выполняться до parse/map, но пользователь не должен отдельно управлять техническим парсингом.
`projectIndex` содержит:
```text
projectIndex
- pages
- html files
- markdown/text files
- css files
- images
- videos
- possible entry points
- ignored files
```
Repeated scan не должен молча перетирать старые решения. Новый scan обязан:
```text
- создать новый scan_version
- попытаться сматчить page/section/media с прошлой версией
- сохранить stable IDs
- показать reconcile summary: matched/changed/new/orphaned/conflicted
```
Пользователь видит найденные страницы и может:
- выбрать страницу для работы;
- исключить технические/лишние файлы;
- переименовать страницу в UI;
- запустить повторный scan.
### 2.3. Project Dashboard
После scan пользователь попадает в dashboard проекта:
```text
Dashboard
- список страниц
- статус baseline audit
- последние SEO runs
- незавершённые working drafts
- последние changesets
- exports
```
## 3. Baseline Audit
Baseline audit — первый смысловой шаг пользователя после создания проекта.
Технически audit использует parse/scan, но в UI пользователь видит именно “Аудит проекта/страницы”.
Проверяется:
```text
page structure:
- title
- description
- h1/h2
- heading order
content:
- видимый текст
- секции
- desktop/adaptive дубли
- потенциальные смысловые блоки
media:
- img/picture/srcset
- video/poster/source
- alt/title/aria
- filename
technical:
- missing meta
- empty headings
- empty alt
- duplicate alt
- obvious weak filenames
```
Результат:
```text
baselineAudit
- project-level issues
- page-level issues
- media issues
- source mapping
```
## 4. Page Workspace
Пользователь выбирает страницу или текстовый источник из проекта.
Для страницы создаются:
```text
originalText
- неизменённый снимок текста из target project
workingText
- рабочая копия для SEO-правок
sectionMap
- sectionId
- DOM selector
- source file
- original text
- working text
- context
```
Codex и пользователь работают только с `workingText`.
Target project не изменяется до ручного `Apply`.
## 5. Meaning Extraction
Codex через `codex exec` анализирует выбранную страницу:
```text
input:
- originalText
- sectionMap
- title/description/h1/h2
- media context
output:
- primary_intent
- section meanings
- product entities
- likely seed queries
- possible competing intents
```
Пользователь видит смысловую карту и может поправить:
- главный смысл;
- смысл секции;
- неверно выделенные сущности;
- лишние seed-запросы.
## 6. Seed Review
UI показывает seed-запросы:
```text
seed
- phrase
- source section
- reason
- confidence
- selected true/false
```
Пользователь:
- включает/выключает seeds;
- добавляет свои;
- объединяет похожие;
- помечает приоритет.
После утверждения seeds идут в Wordstat.
## 7. Wordstat Collection
Wordstat MCP — рука в Wordstat.
Для выбранных seeds собираются:
```text
wordstat result
- seed phrase
- related query
- frequency
- frequency group: high / mid / low
- dynamics
- region if available
```
UI должен позволять:
- запускать сбор по одному seed;
- запускать batch;
- видеть raw results;
- повторять сбор;
- сохранять результаты в истории проекта.
## 8. Keyword Cleaning
Это assisted workflow, не ручная таблица с нуля.
`keywordService` делает:
```text
1. normalize
2. remove duplicates
3. apply project blacklist
4. mark obvious trash
5. group close phrases
6. send ambiguous phrases to Codex classification
7. return suggested statuses
```
Статусы:
```text
use
support
article
risky
trash
```
UI:
```text
Keyword Cleaning screen
- таблица ключей
- частотность
- seed source
- proposed status
- reason
- confidence
- bulk actions
- filters: high/mid/low/use/trash/risky
```
Пользователь утверждает результат галками и массовыми действиями.
## 9. Keyword Map
Keyword Map не должен быть ручной дрочней.
Система предлагает карту:
```text
keyword -> section
keyword -> role
keyword -> limit
keyword -> soft_forms
```
Codex использует:
- section meanings;
- primary_intent;
- keyword statuses;
- частотность;
- текущий текст секции;
- риск переспама.
Роли:
```text
primary
secondary
supporting
do_not_use_on_page
article_backlog
```
UI:
```text
Keyword Map screen
- секции страницы слева
- предложенные ключи справа
- primary/secondary badges
- soft_forms
- exact_limit
- warnings
- approve/apply suggestions
```
Пользователь редактирует предложения, но не раскладывает всё с нуля.
## 10. Optimization Plan
Перед переписыванием система собирает план:
```text
optimizationPlan
- sections to edit
- sections to leave unchanged
- keywords to add
- keywords to reduce
- media SEO tasks
- length risks
- tone risks
```
UI показывает план и просит подтверждение.
Без подтверждения plan не переходит в rewrite.
## 11. Rewrite Workspace
Пользователь выбирает секцию.
UI показывает:
```text
left:
- originalText
- current workingText
right:
- assigned keywords
- section meaning
- limits
- warnings
- Codex variants
```
Codex предлагает варианты, но не меняет target project.
Пользователь может:
- принять вариант;
- смешать варианты;
- отредактировать руками;
- попросить другой тон;
- попросить ужать текст;
- отменить изменения в working draft.
## 12. Media SEO
Media SEO может идти как отдельный экран или как часть плана оптимизации.
Система находит:
```text
img
picture/source/srcset
video/source/poster
CSS background images
filename
alt/title/aria
captions/context
```
Для каждого media item:
```text
status:
ok
missing
weak
duplicate
decorative
needs-human
```
Codex может предложить:
- alt;
- caption;
- video description;
- filename suggestion.
Правило:
```text
ключ можно встроить в alt/filename/caption,
если он естественно описывает изображение и контекст секции.
```
Не набивать ключи без смысла.
## 13. Final Validation
После текстовых и media-правок запускаются:
```text
checkOverseo:
- перетошнота
- заспамленность
- водность
- повторы
- proximity
- keyword stuffing в alt/filename
ru-text:
- тон
- читаемость
- канцелярит
- typographic/editorial issues
structure checks:
- title/description/h1/h2
- section meaning conflict
- design length limit
media checks:
- missing/weak/duplicate alt
- decorative media
- video context
- filename rename plan
```
UI показывает:
- общий статус;
- ошибки;
- предупреждения;
- diff `originalText` vs `workingText`;
- media diff;
- что нужно поправить до apply.
Важно: любые media-правки тоже проходят финальную проверку. Нельзя сделать Media SEO и сразу Apply без Final Validation.
## 14. Apply Changes
Apply — ручной шаг.
Перед применением:
```text
UI shows:
- files affected
- sections affected
- text diff
- media diff
- filename rename plan
- validation status
```
Пользователь нажимает `Apply to target project`.
Приложение:
- создаёт changeset;
- делает dry-run;
- сохраняет backup affected files;
- применяет изменения в target project;
- проверяет rename/update references;
- делает rollback, если критический шаг упал;
- сохраняет отчёт;
- сохраняет run history.
Codex должен получать жёсткое ограничение:
```text
Do not modify application code.
Only modify approved text/media fields in target project:
- visible text
- title/description
- headings
- alt/title/aria
- captions
- approved filenames
```
## 15. Export
После apply пользователь может сделать export:
```text
Export options:
- zip target project
- export reports
- export keyword map
- export changeset summary
```
Это нужно для ручной заливки на хостинг.
Автопубликации нет.
## 16. Project History
В проекте должна сохраняться история:
```text
history:
- scans
- baseline audits
- Wordstat collections
- keyword cleaning decisions
- keyword maps
- optimization plans
- rewrites
- validations
- changesets
- exports
```
Это нужно, чтобы:
- вернуться к работе позже;
- понять, что уже сделано;
- подключить Яндекс Метрику/Вебмастер в будущем;
- делать дальнейшие SEO-корректировки.
## 17. Local Storage
MVP локальный, но проектный.
Рекомендуемая модель:
```text
Postgres:
- projects
- project_sources
- scan_versions
- pages / page_versions
- sections / section_versions
- media_assets / media_versions
- runs
- seeds
- wordstat_jobs / wordstat_results
- keyword_decisions
- keyword_map_items
- drafts / draft_items
- validations / validation_issues
- changesets / changeset_items
- exports
Object storage:
- raw dumps
- reports
- snapshots
- previews / video frames
- backups
- export archives
```
Правило:
```text
Postgres = source of truth
Object storage = artifacts
YAML/JSON/MD = export/debug only
```
## 18. Main UI Navigation
```text
Projects
-> Project Dashboard
-> Scan / Baseline Audit
-> Pages
-> Seeds
-> Wordstat
-> Keyword Cleaning
-> Keyword Map
-> Optimization Plan
-> Rewrite
-> Media SEO
-> Final Validation
-> Apply / Export
-> History / Reports
```
## 19. What Is Not MVP
- создание текстов с нуля;
- автопостинг;
- Яндекс Метрика / Вебмастер;
- полноценная версионность draft history;
- Codex skills;
- облачная версия;
- командная работа.
Эти слои можно добавить позже, когда базовый проектный SEO-цикл стабилен.
## 20. Related Development Guardrails
Технический порядок backend pipeline, зоны ответственности инструментов и риски разработки вынесены в `IMPLEMENTATION_GUARDRAILS.md`.
Важно: этот user flow описывает, как процесс видит пользователь. Внутренний порядок выполнения должен соответствовать guardrails.
UI/UX-набор обязательных экранов, компонентов и состояний вынесен в `UI_UX_SPEC.md`.
Пошаговый план реализации с критериями готовности вынесен в `TASK_MANAGER.md`.

View File

View File

@ -0,0 +1,12 @@
<!doctype html>
<html lang="ru">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>seo_mode</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

View File

@ -0,0 +1,25 @@
{
"name": "@seo-mode/app",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"dev": "vite --host 127.0.0.1",
"build": "tsc -p tsconfig.json && vite build",
"preview": "vite preview --host 127.0.0.1",
"typecheck": "tsc --noEmit -p tsconfig.json"
},
"dependencies": {
"@xyflow/react": "^12.11.1",
"lucide-react": "^0.561.0",
"react": "^19.2.3",
"react-dom": "^19.2.3"
},
"devDependencies": {
"@types/react": "^19.2.7",
"@types/react-dom": "^19.2.3",
"@vitejs/plugin-react": "^5.1.1",
"typescript": "^5.9.3",
"vite": "^8.0.16"
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,9 @@
import { StrictMode } from "react";
import { createRoot } from "react-dom/client";
import { App } from "./App";
createRoot(document.getElementById("root")!).render(
<StrictMode>
<App />
</StrictMode>
);

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1 @@
/// <reference types="vite/client" />

View File

@ -0,0 +1,9 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"jsx": "react-jsx",
"lib": ["ES2022", "DOM", "DOM.Iterable"],
"noEmit": true
},
"include": ["src/**/*.ts", "src/**/*.tsx", "vite.config.ts"]
}

View File

@ -0,0 +1,10 @@
import react from "@vitejs/plugin-react";
import { defineConfig } from "vite";
export default defineConfig({
plugins: [react()],
server: {
port: 5173,
strictPort: false
}
});

View File

@ -0,0 +1,15 @@
# Локальная инфраструктура
Эта папка содержит локальную инфраструктуру разработки для seo_mode.
```powershell
docker compose -f infra/docker-compose.yml up -d
```
Сервисы:
- Postgres: `localhost:5432`
- MinIO API: `localhost:9000`
- MinIO console: `localhost:9001`
Используй `.env.example` как базовый пример локальных переменных окружения.

View File

@ -0,0 +1,39 @@
services:
postgres:
image: postgres:16-alpine
container_name: seo-farm-postgres
environment:
POSTGRES_DB: seo_farm
POSTGRES_USER: seo_farm
POSTGRES_PASSWORD: seo_farm
ports:
- "5432:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U seo_farm -d seo_farm"]
interval: 5s
timeout: 3s
retries: 10
minio:
image: minio/minio:latest
container_name: seo-farm-minio
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: seo_farm
MINIO_ROOT_PASSWORD: seo_farm_secret
ports:
- "9000:9000"
- "9001:9001"
volumes:
- minio-data:/data
healthcheck:
test: ["CMD", "mc", "ready", "local"]
interval: 5s
timeout: 3s
retries: 10
volumes:
postgres-data:
minio-data:

4382
seo_mode/seo_mode/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,21 @@
{
"name": "seo-mode",
"version": "0.1.0",
"private": true,
"type": "module",
"workspaces": [
"app",
"server"
],
"scripts": {
"dev": "concurrently -n app,server -c cyan,green \"npm run dev -w app\" \"npm run dev -w server\"",
"dev:app": "npm run dev -w app",
"dev:server": "npm run dev -w server",
"build": "npm run build -w server && npm run build -w app",
"typecheck": "npm run typecheck -w server && npm run typecheck -w app",
"db:migrate": "npm run db:migrate -w server"
},
"devDependencies": {
"concurrently": "^9.2.1"
}
}

View File

@ -0,0 +1,245 @@
create extension if not exists pgcrypto;
create type source_type as enum ('local_folder', 'git_repo', 'zip_upload', 'site_crawl');
create type run_status as enum ('queued', 'running', 'done', 'failed', 'cancelled');
create type reconcile_status as enum ('matched', 'changed', 'new', 'orphaned', 'conflicted');
create table projects (
id uuid primary key default gen_random_uuid(),
name text not null,
slug text not null unique,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table project_sources (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
type source_type not null,
config jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table scan_versions (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
source_id uuid not null references project_sources(id) on delete cascade,
status run_status not null default 'queued',
project_index_key text,
started_at timestamptz,
completed_at timestamptz,
error_message text,
created_at timestamptz not null default now()
);
create table pages (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
logical_key text not null,
created_at timestamptz not null default now(),
unique(project_id, logical_key)
);
create table page_versions (
id uuid primary key default gen_random_uuid(),
page_id uuid not null references pages(id) on delete cascade,
scan_version_id uuid not null references scan_versions(id) on delete cascade,
source_path text not null,
url_path text,
title text,
description text,
fingerprint text,
reconcile_status reconcile_status not null default 'new',
raw jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table sections (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
page_id uuid references pages(id) on delete cascade,
logical_key text not null,
created_at timestamptz not null default now(),
unique(project_id, logical_key)
);
create table section_versions (
id uuid primary key default gen_random_uuid(),
section_id uuid not null references sections(id) on delete cascade,
scan_version_id uuid not null references scan_versions(id) on delete cascade,
selector text,
heading text,
normalized_text_hash text,
selector_fingerprint text,
reconcile_status reconcile_status not null default 'new',
raw jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table media_assets (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
logical_key text not null,
created_at timestamptz not null default now(),
unique(project_id, logical_key)
);
create table media_versions (
id uuid primary key default gen_random_uuid(),
media_id uuid not null references media_assets(id) on delete cascade,
scan_version_id uuid not null references scan_versions(id) on delete cascade,
source_path text not null,
media_type text not null,
section_id uuid references sections(id) on delete set null,
reconcile_status reconcile_status not null default 'new',
raw jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table runs (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
run_type text not null,
status run_status not null default 'queued',
input jsonb not null default '{}'::jsonb,
output jsonb,
error_message text,
started_at timestamptz,
completed_at timestamptz,
created_at timestamptz not null default now()
);
create table seeds (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
section_id uuid references sections(id) on delete set null,
phrase text not null,
source text not null default 'model',
selected boolean not null default true,
reason text,
confidence numeric(5, 4),
created_at timestamptz not null default now()
);
create table wordstat_jobs (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
seed_id uuid references seeds(id) on delete set null,
status run_status not null default 'queued',
raw_result_key text,
created_at timestamptz not null default now(),
completed_at timestamptz
);
create table wordstat_results (
id uuid primary key default gen_random_uuid(),
job_id uuid not null references wordstat_jobs(id) on delete cascade,
phrase text not null,
frequency integer,
frequency_group text,
raw jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table keyword_decisions (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
wordstat_result_id uuid references wordstat_results(id) on delete set null,
phrase text not null,
status text not null,
reason text,
approved boolean not null default false,
created_at timestamptz not null default now()
);
create table keyword_map_items (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
section_id uuid references sections(id) on delete cascade,
keyword_decision_id uuid references keyword_decisions(id) on delete cascade,
role text not null,
exact_limit integer,
soft_forms jsonb not null default '[]'::jsonb,
approved boolean not null default false,
created_at timestamptz not null default now()
);
create table optimization_plans (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
payload jsonb not null default '{}'::jsonb,
approved boolean not null default false,
created_at timestamptz not null default now()
);
create table drafts (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
page_id uuid references pages(id) on delete cascade,
status text not null default 'active',
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table draft_items (
id uuid primary key default gen_random_uuid(),
draft_id uuid not null references drafts(id) on delete cascade,
section_id uuid references sections(id) on delete cascade,
field text not null,
original_snapshot_key text,
working_snapshot_key text,
working_text text,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);
create table validations (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
draft_id uuid references drafts(id) on delete set null,
status text not null,
summary jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table validation_issues (
id uuid primary key default gen_random_uuid(),
validation_id uuid not null references validations(id) on delete cascade,
severity text not null,
code text not null,
message text not null,
source_ref jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now()
);
create table changesets (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
draft_id uuid references drafts(id) on delete set null,
validation_id uuid references validations(id) on delete set null,
status text not null default 'pending',
backup_key text,
summary jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
applied_at timestamptz
);
create table changeset_items (
id uuid primary key default gen_random_uuid(),
changeset_id uuid not null references changesets(id) on delete cascade,
change_type text not null,
source_path text not null,
selector_or_media_id text,
before_value text,
after_value text,
created_at timestamptz not null default now()
);
create table exports (
id uuid primary key default gen_random_uuid(),
project_id uuid not null references projects(id) on delete cascade,
export_type text not null,
object_key text not null,
created_at timestamptz not null default now()
);

View File

@ -0,0 +1 @@
alter type source_type add value if not exists 'browser_folder';

View File

@ -0,0 +1 @@
create unique index if not exists projects_name_unique_ci on projects (lower(name));

View File

@ -0,0 +1,2 @@
alter table pages
add column if not exists selected boolean not null default false;

View File

@ -0,0 +1,36 @@
alter table seeds
add column if not exists semantic_run_id uuid references runs(id) on delete set null,
add column if not exists cluster_id text,
add column if not exists cluster_title text,
add column if not exists priority text,
add column if not exists updated_at timestamptz not null default now();
create unique index if not exists seeds_project_semantic_phrase_cluster_idx
on seeds(project_id, semantic_run_id, lower(phrase), coalesce(cluster_id, ''));
alter table wordstat_jobs
add column if not exists semantic_run_id uuid references runs(id) on delete set null,
add column if not exists provider text not null default 'disabled',
add column if not exists mode text not null default 'disabled',
add column if not exists region text not null default 'ru',
add column if not exists requested_phrases jsonb not null default '[]'::jsonb,
add column if not exists result_count integer not null default 0,
add column if not exists error_message text,
add column if not exists started_at timestamptz,
add column if not exists updated_at timestamptz not null default now();
create index if not exists wordstat_jobs_project_semantic_created_idx
on wordstat_jobs(project_id, semantic_run_id, created_at desc);
alter table wordstat_results
add column if not exists seed_id uuid references seeds(id) on delete set null,
add column if not exists source_phrase text,
add column if not exists normalized_phrase text,
add column if not exists region text,
add column if not exists position integer;
create index if not exists wordstat_results_job_seed_idx
on wordstat_results(job_id, seed_id);
create index if not exists wordstat_results_phrase_idx
on wordstat_results(lower(phrase));

View File

@ -0,0 +1,9 @@
create table if not exists external_service_settings (
service_id text primary key,
enabled boolean not null default false,
mode text not null default 'disabled',
public_config jsonb not null default '{}'::jsonb,
secret_config jsonb not null default '{}'::jsonb,
created_at timestamptz not null default now(),
updated_at timestamptz not null default now()
);

View File

@ -0,0 +1,30 @@
{
"name": "@seo-mode/server",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"dev": "tsx watch src/index.ts",
"build": "tsc -p tsconfig.json",
"start": "node dist/index.js",
"typecheck": "tsc --noEmit -p tsconfig.json",
"db:migrate": "tsx src/scripts/migrate.ts"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.946.0",
"cors": "^2.8.5",
"dotenv": "^17.2.3",
"express": "^5.2.1",
"pg": "^8.16.3",
"playwright-core": "^1.61.1",
"zod": "^4.1.13"
},
"devDependencies": {
"@types/cors": "^2.8.19",
"@types/express": "^5.0.6",
"@types/node": "^24.10.2",
"@types/pg": "^8.16.0",
"tsx": "^4.21.0",
"typescript": "^5.9.3"
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,73 @@
import "dotenv/config";
import { z } from "zod";
const booleanFromString = z.preprocess((value) => {
if (typeof value === "boolean") return value;
if (typeof value !== "string") return value;
return ["1", "true", "yes"].includes(value.toLowerCase());
}, z.boolean());
const envSchema = z.object({
SERVER_PORT: z.coerce.number().int().positive().default(4100),
DATABASE_URL: z.string().url().default("postgres://seo_farm:seo_farm@localhost:5432/seo_farm"),
S3_ENDPOINT: z.string().url().default("http://localhost:9000"),
S3_REGION: z.string().default("us-east-1"),
S3_ACCESS_KEY_ID: z.string().default("seo_farm"),
S3_SECRET_ACCESS_KEY: z.string().default("seo_farm_secret"),
S3_BUCKET: z.string().default("seo-farm-artifacts"),
S3_FORCE_PATH_STYLE: booleanFromString.default(true),
WORDSTAT_PROVIDER: z.enum(["disabled", "mcp_kv", "yandex_api"]).default("disabled"),
WORDSTAT_MCP_ENDPOINT: z.string().url().optional(),
WORDSTAT_API_BASE_URL: z.string().url().optional(),
WORDSTAT_API_TOKEN: z.string().optional(),
WORDSTAT_AUTH_TYPE: z.enum(["api_key", "iam_token"]).default("api_key"),
WORDSTAT_FOLDER_ID: z.string().optional(),
WORDSTAT_REGION_IDS: z.string().default("225"),
WORDSTAT_DEVICES: z.string().default("DEVICE_ALL"),
WORDSTAT_NUM_PHRASES: z.coerce.number().int().positive().default(50),
SERP_PROVIDER: z.enum(["disabled", "external_api"]).default("disabled"),
SERP_API_BASE_URL: z.string().url().optional(),
SERP_API_TOKEN: z.string().optional(),
PREVIEW_CHROME_PATH: z.string().optional(),
PREVIEW_SNAPSHOT_CONCURRENCY: z.coerce.number().int().positive().max(4).default(2),
PREVIEW_SNAPSHOT_WAIT_MS: z.coerce.number().int().nonnegative().max(10_000).default(1400)
});
const parsedEnv = envSchema.parse(process.env);
export const env = {
serverPort: parsedEnv.SERVER_PORT,
databaseUrl: parsedEnv.DATABASE_URL,
s3: {
endpoint: parsedEnv.S3_ENDPOINT,
region: parsedEnv.S3_REGION,
accessKeyId: parsedEnv.S3_ACCESS_KEY_ID,
secretAccessKey: parsedEnv.S3_SECRET_ACCESS_KEY,
bucket: parsedEnv.S3_BUCKET,
forcePathStyle: parsedEnv.S3_FORCE_PATH_STYLE
},
marketProviders: {
wordstat: {
provider: parsedEnv.WORDSTAT_PROVIDER,
mcpEndpoint: parsedEnv.WORDSTAT_MCP_ENDPOINT,
apiBaseUrl: parsedEnv.WORDSTAT_API_BASE_URL,
apiToken: parsedEnv.WORDSTAT_API_TOKEN,
authType: parsedEnv.WORDSTAT_AUTH_TYPE,
folderId: parsedEnv.WORDSTAT_FOLDER_ID,
regionIds: parsedEnv.WORDSTAT_REGION_IDS.split(",").map((item) => item.trim()).filter(Boolean),
devices: parsedEnv.WORDSTAT_DEVICES.split(",").map((item) => item.trim()).filter(Boolean),
numPhrases: parsedEnv.WORDSTAT_NUM_PHRASES,
hasApiToken: Boolean(parsedEnv.WORDSTAT_API_TOKEN)
},
serp: {
provider: parsedEnv.SERP_PROVIDER,
apiBaseUrl: parsedEnv.SERP_API_BASE_URL,
hasApiToken: Boolean(parsedEnv.SERP_API_TOKEN)
}
},
preview: {
chromePath: parsedEnv.PREVIEW_CHROME_PATH || "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
snapshotConcurrency: parsedEnv.PREVIEW_SNAPSHOT_CONCURRENCY,
snapshotWaitMs: parsedEnv.PREVIEW_SNAPSHOT_WAIT_MS
}
};

View File

@ -0,0 +1,15 @@
import pg from "pg";
import { env } from "../config/env.js";
export const pool = new pg.Pool({
connectionString: env.databaseUrl
});
export async function checkDatabase() {
const result = await pool.query<{ now: Date }>("select now() as now");
return result.rows[0];
}
export async function closeDatabase() {
await pool.end();
}

View File

@ -0,0 +1,53 @@
import cors from "cors";
import express from "express";
import { checkDatabase } from "../db/client.js";
import { previewRouter } from "../preview/routes.js";
import { projectsRouter } from "../projects/routes.js";
import { settingsRouter } from "../settings/routes.js";
import { storageProvider } from "../storage/index.js";
export function createServer() {
const app = express();
app.use(cors({ exposedHeaders: ["Content-Disposition"] }));
app.use(express.json({ limit: "150mb" }));
app.use("/projects", projectsRouter);
app.use("/preview", previewRouter);
app.use("/settings", settingsRouter);
app.get("/health", (_request, response) => {
response.json({
status: "ok",
service: "seo_mode-server"
});
});
app.get("/health/deep", async (_request, response) => {
const checks: Record<string, unknown> = {};
try {
checks.database = await checkDatabase();
} catch (error) {
checks.database = { error: error instanceof Error ? error.message : "unknown error" };
}
try {
await storageProvider.ensureReady();
checks.objectStorage = { status: "ok" };
} catch (error) {
checks.objectStorage = { error: error instanceof Error ? error.message : "unknown error" };
}
const hasError = Object.values(checks).some((value) => {
return typeof value === "object" && value !== null && "error" in value;
});
response.status(hasError ? 503 : 200).json({
status: hasError ? "degraded" : "ok",
checks
});
});
return app;
}

View File

@ -0,0 +1,8 @@
import { env } from "./config/env.js";
import { createServer } from "./http/server.js";
const app = createServer();
app.listen(env.serverPort, () => {
console.log(`seo_mode server listening on http://localhost:${env.serverPort}`);
});

View File

@ -0,0 +1,372 @@
import { env } from "../config/env.js";
import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js";
import { collectWordstatEvidence, getLatestWordstatEvidence, type WordstatEvidence } from "./wordstatRepository.js";
import { createWordstatProvider } from "./wordstatProvider.js";
type MarketProviderId = "model" | "serp" | "text_quality" | "wordstat";
type MarketProviderStatus = "connected" | "disabled" | "not_configured" | "not_implemented";
type MarketEnrichmentState = "not_ready" | "not_collected" | "not_configured" | "partial_ready";
type MarketEvidenceStatus =
| "collection_failed"
| "collected"
| "collected_no_signal"
| "collected_related_only"
| "deterministic_only"
| "not_collected"
| "not_configured"
| "ready_for_collection";
type MarketProviderDescriptor = {
id: MarketProviderId;
label: string;
role: string;
status: MarketProviderStatus;
mode: string;
userActionRequired: boolean;
platformActionRequired: boolean;
message: string;
};
export type MarketEnrichmentContract = {
schemaVersion: "market-enrichment.v1";
projectId: string;
semanticRunId: string | null;
generatedAt: string;
state: MarketEnrichmentState;
readiness: {
canCollectWordstat: boolean;
canCollectSerp: boolean;
seedCount: number;
briefCount: number;
configuredProviderCount: number;
userSetupRequired: boolean;
};
providers: MarketProviderDescriptor[];
seedQueue: Array<{
phrase: string;
clusterId: string;
clusterTitle: string;
priority: "high" | "medium" | "low";
source: "detected" | "ontology";
evidenceStatus: MarketEvidenceStatus;
targetProviders: MarketProviderId[];
blocker: string | null;
frequency: number | null;
frequencyGroup: string | null;
relatedCount: number;
lastCollectedAt: string | null;
}>;
briefEvidence: Array<{
path: string;
priority: "high" | "medium" | "low";
action: "create" | "strengthen" | "validate";
qualityScore: number | null;
qualityLevel: "blocked" | "needs_work" | "ready" | null;
seedPhrases: string[];
evidenceStatus: MarketEvidenceStatus;
requiredProviders: MarketProviderId[];
missingEvidence: string[];
}>;
wordstat: WordstatEvidence;
nextActions: string[];
};
function isSerpConfigured() {
const config = env.marketProviders.serp;
return config.provider !== "disabled" && Boolean(config.apiBaseUrl && config.hasApiToken);
}
async function buildProviderRegistry(): Promise<MarketProviderDescriptor[]> {
const serpConfig = env.marketProviders.serp;
const wordstatStatus = (await createWordstatProvider()).getStatus();
const serpConfigured = isSerpConfigured();
return [
{
id: "wordstat",
label: "Yandex Wordstat",
role: "Частотность, похожие запросы, региональные и сезонные сигналы по seed-фразам.",
status: wordstatStatus.connected ? "connected" : "not_configured",
mode: wordstatStatus.mode,
userActionRequired: false,
platformActionRequired: !wordstatStatus.connected,
message: wordstatStatus.connected
? wordstatStatus.message
: `${wordstatStatus.message} Пользовательский сценарий не должен требовать ручной настройки MCP.`
},
{
id: "serp",
label: "Yandex SERP",
role: "Типы страниц в выдаче, конкурентные углы, сложность и сниппеты.",
status: serpConfigured ? "connected" : "not_configured",
mode: serpConfig.provider,
userActionRequired: false,
platformActionRequired: !serpConfigured,
message: serpConfigured
? "SERP provider настроен; можно собирать конкурентные evidence."
: "SERP provider ещё не настроен; deterministic analysis не должен притворяться рыночной проверкой."
},
{
id: "model",
label: "Model Provider",
role: "Классификация неоднозначных фраз, интентов и конкурентных формулировок.",
status: "not_implemented",
mode: "provider_abstraction",
userActionRequired: false,
platformActionRequired: true,
message: "Model provider будет подключаться платформенно через общий AI layer; в MVP решение не просит пользователя о ключах."
},
{
id: "text_quality",
label: "Text Quality",
role: "Локальная проверка заспамленности, воды и перебора ключей после рерайта.",
status: "not_implemented",
mode: "local_ru_text",
userActionRequired: false,
platformActionRequired: true,
message: "Text.ru API не подключается; по ТЗ нужен локальный quality checker по аналогичной модели показателей."
}
];
}
function getProviderStatus(providers: MarketProviderDescriptor[], id: MarketProviderId) {
return providers.find((provider) => provider.id === id)?.status ?? "not_configured";
}
function getEvidenceStatus(providers: MarketProviderDescriptor[], requiredProviders: MarketProviderId[]): MarketEvidenceStatus {
if (requiredProviders.length === 0) {
return "deterministic_only";
}
const statuses = requiredProviders.map((providerId) => getProviderStatus(providers, providerId));
if (statuses.some((status) => status === "not_configured" || status === "disabled")) {
return "not_configured";
}
if (statuses.every((status) => status === "connected")) {
return "ready_for_collection";
}
return "not_collected";
}
function getSeedBlocker(evidenceStatus: MarketEvidenceStatus) {
if (evidenceStatus === "not_configured") {
return "Нужен платформенный Wordstat provider; пользователь не должен вручную подключать MCP или внешний аккаунт.";
}
if (evidenceStatus === "collection_failed") {
return "Последний Wordstat job упал; смотри provider error и raw job status в dev-mode.";
}
if (
evidenceStatus === "ready_for_collection" ||
evidenceStatus === "collected" ||
evidenceStatus === "collected_related_only" ||
evidenceStatus === "collected_no_signal"
) {
return null;
}
return "Сбор ещё не запускался.";
}
function normalizePhrase(value: string) {
return value.trim().replace(/\s+/g, " ").toLowerCase();
}
function buildWordstatSeedMap(wordstat: WordstatEvidence) {
return new Map(wordstat.seedResults.map((result) => [normalizePhrase(result.phrase), result]));
}
function getSeedEvidenceStatus(
providers: MarketProviderDescriptor[],
wordstat: WordstatEvidence,
wordstatResult: WordstatEvidence["seedResults"][number] | undefined
): MarketEvidenceStatus {
if (wordstatResult?.frequency !== null && wordstatResult?.frequency !== undefined) {
return "collected";
}
if (wordstat.latestJob?.status === "failed") {
return "collection_failed";
}
if (wordstat.latestJob?.status === "done" && wordstatResult && wordstatResult.relatedCount > 0) {
return "collected_related_only";
}
if (wordstat.latestJob?.status === "done") {
return "collected_no_signal";
}
return getEvidenceStatus(providers, ["wordstat"]);
}
function buildSeedQueue(analysis: SemanticAnalysisRun, providers: MarketProviderDescriptor[], wordstat: WordstatEvidence) {
const wordstatResults = buildWordstatSeedMap(wordstat);
return analysis.seedCandidates.slice(0, 80).map((seed) => {
const targetProviders: MarketProviderId[] = ["wordstat"];
const result = wordstatResults.get(normalizePhrase(seed.phrase));
const evidenceStatus = getSeedEvidenceStatus(providers, wordstat, result);
return {
phrase: seed.phrase,
clusterId: seed.clusterId,
clusterTitle: seed.clusterTitle,
priority: seed.priority,
source: seed.source,
evidenceStatus,
targetProviders,
blocker: getSeedBlocker(evidenceStatus),
frequency: result?.frequency ?? null,
frequencyGroup: result?.frequencyGroup ?? null,
relatedCount: result?.relatedCount ?? 0,
lastCollectedAt: result?.collectedAt ?? null
};
});
}
function buildBriefEvidence(analysis: SemanticAnalysisRun, providers: MarketProviderDescriptor[]) {
return (analysis.landingPageBriefs ?? []).slice(0, 80).map((brief) => {
const requiredProviders: MarketProviderId[] = ["wordstat", "serp"];
const evidenceStatus = getEvidenceStatus(providers, requiredProviders);
return {
path: brief.path,
priority: brief.priority,
action: brief.action,
qualityScore: brief.qualityContract?.score ?? null,
qualityLevel: brief.qualityContract?.level ?? null,
seedPhrases: brief.seedPhrases.slice(0, 8),
evidenceStatus,
requiredProviders,
missingEvidence: brief.qualityContract?.missingEvidence ?? [
"Wordstat частотность и сезонность",
"SERP-конкуренты и типы страниц",
"Региональная выдача"
]
};
});
}
function getEnrichmentState(
analysis: SemanticAnalysisRun | null,
providers: MarketProviderDescriptor[],
seedCount: number,
wordstat: WordstatEvidence
): MarketEnrichmentState {
if (!analysis) {
return "not_ready";
}
if (seedCount === 0) {
return "not_ready";
}
if (getProviderStatus(providers, "wordstat") !== "connected") {
return "not_configured";
}
if (wordstat.latestJob?.status === "done" && wordstat.summary.collectedSeedCount > 0) {
return getProviderStatus(providers, "serp") === "connected" ? "not_collected" : "partial_ready";
}
if (getProviderStatus(providers, "serp") !== "connected") {
return "partial_ready";
}
return "not_collected";
}
function buildNextActions(contract: Omit<MarketEnrichmentContract, "nextActions">) {
const actions: string[] = [];
if (!contract.semanticRunId) {
actions.push("Сначала запустить semantic analysis: без seed candidates market layer не стартует.");
return actions;
}
const wordstat = contract.providers.find((provider) => provider.id === "wordstat");
const serp = contract.providers.find((provider) => provider.id === "serp");
if (wordstat?.status !== "connected") {
actions.push("Настроить платформенный Wordstat provider: MCP-KV endpoint или прямой API adapter на backend.");
} else if (contract.wordstat.latestJob?.status === "done") {
actions.push(
`Wordstat evidence собрано: ${contract.wordstat.summary.collectedSeedCount} seed-фраз, ${contract.wordstat.summary.resultCount} rows. Следующий слой — keywordService cleaning.`
);
} else if (contract.wordstat.latestJob?.status === "failed") {
actions.push("Проверить Wordstat provider: последний job завершился ошибкой, raw/error сохранены в dev-mode contract.");
} else if (contract.readiness.seedCount > 0) {
actions.push(`Запустить Wordstat collection для ${contract.readiness.seedCount} seed-фраз.`);
}
if (serp?.status !== "connected") {
actions.push("Настроить платформенный SERP provider для конкурентных evidence, без ручной работы пользователя.");
}
if (wordstat?.status !== "connected" && contract.readiness.seedCount > 0) {
actions.push(`После настройки провайдера отправить ${contract.readiness.seedCount} seed-фраз в Wordstat queue.`);
}
actions.push("Хранить внешние результаты как evidence при seed/brief/page, а не смешивать их с deterministic-анализом.");
return actions;
}
export async function getMarketEnrichmentContract(projectId: string): Promise<MarketEnrichmentContract> {
const analysis = await getLatestSemanticAnalysis(projectId);
const providers = await buildProviderRegistry();
const wordstat = await getLatestWordstatEvidence(projectId, analysis?.runId ?? null);
const seedQueue = analysis ? buildSeedQueue(analysis, providers, wordstat) : [];
const briefEvidence = analysis ? buildBriefEvidence(analysis, providers) : [];
const configuredProviderCount = providers.filter((provider) => provider.status === "connected").length;
const contractWithoutActions: Omit<MarketEnrichmentContract, "nextActions"> = {
schemaVersion: "market-enrichment.v1",
projectId,
semanticRunId: analysis?.runId ?? null,
generatedAt: new Date().toISOString(),
state: getEnrichmentState(analysis, providers, seedQueue.length, wordstat),
readiness: {
canCollectWordstat:
getProviderStatus(providers, "wordstat") === "connected" &&
seedQueue.length > 0 &&
wordstat.latestJob?.status !== "running",
canCollectSerp: getProviderStatus(providers, "serp") === "connected" && briefEvidence.length > 0,
seedCount: seedQueue.length,
briefCount: briefEvidence.length,
configuredProviderCount,
userSetupRequired: providers.some((provider) => provider.userActionRequired)
},
providers,
seedQueue,
briefEvidence,
wordstat
};
return {
...contractWithoutActions,
nextActions: buildNextActions(contractWithoutActions)
};
}
export async function runMarketEnrichment(projectId: string): Promise<MarketEnrichmentContract> {
const analysis = await getLatestSemanticAnalysis(projectId);
const providers = await buildProviderRegistry();
if (!analysis || getProviderStatus(providers, "wordstat") !== "connected" || analysis.seedCandidates.length === 0) {
return getMarketEnrichmentContract(projectId);
}
try {
await collectWordstatEvidence(projectId, analysis);
} catch (error) {
console.error(error);
}
return getMarketEnrichmentContract(projectId);
}

View File

@ -0,0 +1,370 @@
import { getWordstatRuntimeConfig, type WordstatRuntimeConfig } from "../settings/externalServiceSettings.js";
export type WordstatSeedInput = {
seedId: string;
phrase: string;
clusterId: string;
clusterTitle: string;
priority: "high" | "medium" | "low";
source: "detected" | "ontology";
};
export type WordstatCollectRequest = {
projectId: string;
semanticRunId: string;
region: string;
seeds: WordstatSeedInput[];
};
export type WordstatRelatedQuery = {
phrase: string;
frequency: number | null;
frequencyGroup: string | null;
raw: unknown;
};
export type WordstatSeedResult = {
seedId: string;
seedPhrase: string;
frequency: number | null;
frequencyGroup: string | null;
relatedQueries: WordstatRelatedQuery[];
raw: unknown;
};
export type WordstatCollectResult = {
provider: "mcp_kv" | "yandex_api";
mode: string;
region: string;
collectedAt: string;
seedResults: WordstatSeedResult[];
raw: unknown;
};
export type WordstatProviderStatus =
| {
connected: true;
provider: "mcp_kv" | "yandex_api";
mode: string;
message: string;
}
| {
connected: false;
provider: "disabled" | "mcp_kv" | "yandex_api";
mode: string;
message: string;
};
export interface WordstatProvider {
getStatus(): WordstatProviderStatus;
collect(request: WordstatCollectRequest): Promise<WordstatCollectResult>;
}
export class WordstatProviderUnavailableError extends Error {
constructor(message: string) {
super(message);
this.name = "WordstatProviderUnavailableError";
}
}
function getFrequencyGroup(frequency: number | null, explicitGroup?: unknown) {
if (typeof explicitGroup === "string" && explicitGroup.trim()) {
return explicitGroup.trim();
}
if (frequency === null) {
return null;
}
if (frequency >= 10_000) {
return "high";
}
if (frequency >= 1_000) {
return "mid";
}
return "low";
}
function getString(value: unknown) {
return typeof value === "string" ? value.trim() : "";
}
function getNumber(value: unknown) {
if (typeof value === "number" && Number.isFinite(value)) {
return Math.max(0, Math.round(value));
}
if (typeof value === "string") {
const normalized = Number(value.replace(/\s+/g, ""));
if (Number.isFinite(normalized)) {
return Math.max(0, Math.round(normalized));
}
}
return null;
}
function asRecord(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
}
function asArray(value: unknown) {
return Array.isArray(value) ? value : [];
}
function normalizeRelatedQuery(value: unknown): WordstatRelatedQuery | null {
const record = asRecord(value);
const phrase =
getString(record.phrase) ||
getString(record.query) ||
getString(record.keyword) ||
getString(record.text) ||
getString(record.name);
if (!phrase) {
return null;
}
const frequency =
getNumber(record.frequency) ?? getNumber(record.shows) ?? getNumber(record.count) ?? getNumber(record.impressions);
return {
phrase,
frequency,
frequencyGroup: getFrequencyGroup(frequency, record.frequencyGroup ?? record.frequency_group ?? record.group),
raw: value
};
}
function normalizeSeedResult(value: unknown, seed: WordstatSeedInput): WordstatSeedResult {
const record = asRecord(value);
const phrase =
getString(record.seedPhrase) ||
getString(record.seed_phrase) ||
getString(record.phrase) ||
getString(record.query) ||
seed.phrase;
const frequency =
getNumber(record.frequency) ??
getNumber(record.shows) ??
getNumber(record.count) ??
getNumber(record.total) ??
getNumber(record.impressions);
const relatedSource =
record.relatedQueries ??
record.related_queries ??
record.queries ??
record.items ??
record.results ??
record.associations;
return {
seedId: seed.seedId,
seedPhrase: phrase,
frequency,
frequencyGroup: getFrequencyGroup(frequency, record.frequencyGroup ?? record.frequency_group ?? record.group),
relatedQueries: asArray(relatedSource).flatMap((related) => {
const normalized = normalizeRelatedQuery(related);
return normalized ? [normalized] : [];
}),
raw: value
};
}
function normalizeProviderResponse(
provider: "mcp_kv" | "yandex_api",
mode: string,
region: string,
seeds: WordstatSeedInput[],
payload: unknown
): WordstatCollectResult {
const record = asRecord(payload);
const resultSource = record.results ?? record.seedResults ?? record.seed_results ?? record.data ?? record.items;
const resultRecords = asArray(resultSource);
const resultByPhrase = new Map<string, unknown>();
for (const resultRecord of resultRecords) {
const result = asRecord(resultRecord);
const phrase =
getString(result.seedPhrase) ||
getString(result.seed_phrase) ||
getString(result.phrase) ||
getString(result.query) ||
getString(result.keyword);
if (phrase) {
resultByPhrase.set(phrase.toLowerCase(), resultRecord);
}
}
return {
provider,
mode,
region,
collectedAt: new Date().toISOString(),
seedResults: seeds.map((seed, index) =>
normalizeSeedResult(resultByPhrase.get(seed.phrase.toLowerCase()) ?? resultRecords[index] ?? {}, seed)
),
raw: payload
};
}
class DisabledWordstatProvider implements WordstatProvider {
getStatus(): WordstatProviderStatus {
return {
connected: false,
provider: "disabled",
mode: "disabled",
message: "Wordstat provider отключён на платформе."
};
}
async collect(): Promise<WordstatCollectResult> {
throw new WordstatProviderUnavailableError("Wordstat provider отключён на платформе.");
}
}
async function postJson(endpoint: string, body: unknown, headers: Record<string, string> = {}) {
const response = await fetch(endpoint, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers
},
body: JSON.stringify(body)
});
if (!response.ok) {
const message = await response.text().catch(() => "");
throw new Error(`Wordstat provider вернул ${response.status}${message ? `: ${message.slice(0, 240)}` : ""}`);
}
return response.json() as Promise<unknown>;
}
class McpKvWordstatProvider implements WordstatProvider {
constructor(private readonly endpoint: string | undefined) {}
getStatus(): WordstatProviderStatus {
if (!this.endpoint) {
return {
connected: false,
provider: "mcp_kv",
mode: "mcp_kv",
message: "WORDSTAT_MCP_ENDPOINT не настроен."
};
}
return {
connected: true,
provider: "mcp_kv",
mode: "mcp_kv",
message: "Wordstat MCP-KV endpoint настроен на платформе."
};
}
async collect(request: WordstatCollectRequest): Promise<WordstatCollectResult> {
if (!this.endpoint) {
throw new WordstatProviderUnavailableError("WORDSTAT_MCP_ENDPOINT не настроен.");
}
const payload = await postJson(this.endpoint, {
projectId: request.projectId,
semanticRunId: request.semanticRunId,
region: request.region,
phrases: request.seeds.map((seed) => seed.phrase),
seeds: request.seeds
});
return normalizeProviderResponse("mcp_kv", "mcp_kv", request.region, request.seeds, payload);
}
}
class YandexApiWordstatProvider implements WordstatProvider {
constructor(private readonly config: Extract<WordstatRuntimeConfig, { provider: "yandex_api" }> | null) {}
getStatus(): WordstatProviderStatus {
if (!this.config) {
return {
connected: false,
provider: "yandex_api",
mode: "yandex_search_api",
message: "Yandex Wordstat Search API не настроен: нужен folderId и API key или IAM token."
};
}
return {
connected: true,
provider: "yandex_api",
mode: "yandex_search_api",
message: "Yandex Wordstat Search API настроен на платформе."
};
}
async collect(request: WordstatCollectRequest): Promise<WordstatCollectResult> {
if (!this.config) {
throw new WordstatProviderUnavailableError("Yandex Wordstat Search API не настроен.");
}
const endpoint = `${this.config.apiBaseUrl.replace(/\/+$/, "")}/topRequests`;
const authorization =
this.config.authType === "iam_token" ? `Bearer ${this.config.apiToken}` : `Api-Key ${this.config.apiToken}`;
const seedResponses = [];
for (const seed of request.seeds) {
const payload = await postJson(
endpoint,
{
devices: this.config.devices,
folderId: this.config.folderId,
numPhrases: this.config.numPhrases,
phrase: seed.phrase,
regions: this.config.regionIds
},
{
Authorization: authorization
}
);
const record = asRecord(payload);
const results = asArray(record.results ?? record.topRequests ?? record.items);
const associations = asArray(record.associations ?? record.relatedQueries ?? record.related_queries);
const exactMatch = results
.map((item) => normalizeRelatedQuery(item))
.find((item) => item?.phrase.toLowerCase() === seed.phrase.toLowerCase());
seedResponses.push({
phrase: seed.phrase,
frequency: exactMatch?.frequency ?? null,
relatedQueries: [...results, ...associations],
raw: payload
});
}
return normalizeProviderResponse(
"yandex_api",
"yandex_search_api",
this.config.regionIds.join(","),
request.seeds,
{
results: seedResponses
}
);
}
}
export async function createWordstatProvider(): Promise<WordstatProvider> {
const config = await getWordstatRuntimeConfig();
if (config.provider === "mcp_kv") {
return new McpKvWordstatProvider(config.mcpEndpoint);
}
if (config.provider === "yandex_api") {
return new YandexApiWordstatProvider(config);
}
return new DisabledWordstatProvider();
}

View File

@ -0,0 +1,693 @@
import { pool } from "../db/client.js";
import { storageProvider } from "../storage/index.js";
import type { SemanticAnalysisRun } from "../analysis/semanticAnalysis.js";
import {
createWordstatProvider,
type WordstatCollectResult,
type WordstatSeedInput,
WordstatProviderUnavailableError
} from "./wordstatProvider.js";
type SeedRow = {
id: string;
phrase: string;
cluster_id: string | null;
cluster_title: string | null;
priority: "high" | "medium" | "low" | null;
source: "detected" | "ontology" | string;
};
type WordstatJobRow = {
id: string;
project_id: string;
semantic_run_id: string | null;
status: "queued" | "running" | "done" | "failed" | "cancelled";
provider: string;
mode: string;
region: string;
requested_phrases: string[];
raw_result_key: string | null;
result_count: number;
error_message: string | null;
started_at: Date | null;
completed_at: Date | null;
created_at: Date;
updated_at: Date;
};
type WordstatResultRow = {
id: string;
job_id: string;
seed_id: string | null;
phrase: string;
source_phrase: string | null;
normalized_phrase: string | null;
frequency: number | null;
frequency_group: string | null;
region: string | null;
position: number | null;
raw: Record<string, unknown>;
created_at: Date;
};
export type WordstatEvidence = {
latestJob: {
id: string;
status: WordstatJobRow["status"];
provider: string;
mode: string;
region: string;
requestedPhraseCount: number;
rawResultKey: string | null;
resultCount: number;
errorMessage: string | null;
startedAt: string | null;
completedAt: string | null;
createdAt: string;
updatedAt: string;
} | null;
seedResults: Array<{
seedId: string | null;
phrase: string;
sourcePhrase: string | null;
frequency: number | null;
frequencyGroup: string | null;
region: string | null;
relatedCount: number;
collectedAt: string;
}>;
topResults: Array<{
phrase: string;
sourcePhrase: string | null;
frequency: number | null;
frequencyGroup: string | null;
region: string | null;
}>;
summary: {
collectedSeedCount: number;
exactFrequencySeedCount: number;
relatedOnlySeedCount: number;
noSignalSeedCount: number;
resultCount: number;
totalFrequency: number;
highFrequencyCount: number;
midFrequencyCount: number;
lowFrequencyCount: number;
};
};
function toIsoDate(value: Date | null) {
return value ? value.toISOString() : null;
}
function normalizePhrase(value: string) {
return value.trim().replace(/\s+/g, " ").toLowerCase();
}
const RELATED_QUERY_STOP_WORDS = new Set([
"a",
"an",
"and",
"by",
"for",
"in",
"of",
"on",
"or",
"the",
"to",
"без",
"в",
"для",
"и",
"или",
"как",
"на",
"о",
"об",
"от",
"по",
"при",
"с",
"со",
"что",
"это"
]);
const RELATED_QUERY_BLOCKLIST = [
/\bbusiness\s+cats?\b/i,
/\bcats?\b/i,
/(^|\s)асу(\s|$)/i,
/асу\s*тп/i,
/расписан/i,
/теори[яи]\s+автоматическ/i,
/как\s+сделать\s+автоматическ\w*\s+содержан/i,
/автоматическ\w*\s+содержан/i
];
const PRODUCT_SIGNAL_PATTERNS = [
/\b(ai|ии|bpm|crm|api|workflow)\b/i,
/1с/i,
/агент/i,
/ассистент/i,
/автоматизац/i,
/бизнес/i,
/данн/i,
/закуп/i,
/интеграц/i,
/интеллект/i,
/инфраструктур/i,
/искусствен/i,
/облак/i,
/операц/i,
/оркестр/i,
/платформ/i,
/процесс/i,
/проект/i,
/разработ/i,
/роботизац/i,
/тендер/i,
/цифров/i,
/bim/i,
/on[-\s]?premise/i
];
const AUTOMATION_SIGNAL_PATTERN = /автоматизац|роботизац|\bbpm\b|\bworkflow\b|процесс/i;
function stemToken(token: string) {
if (/^[a-z0-9]+$/i.test(token)) {
return token.toLowerCase();
}
return token
.toLocaleLowerCase("ru-RU")
.replace(/(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i, "");
}
function getMeaningfulTokenStems(phrase: string) {
return new Set(
phrase
.toLocaleLowerCase("ru-RU")
.replace(/ё/g, "е")
.split(/[^a-zа-я0-9]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 1 && !RELATED_QUERY_STOP_WORDS.has(token))
.map(stemToken)
.filter((token) => token.length > 1 && !RELATED_QUERY_STOP_WORDS.has(token))
);
}
function countIntersection(left: Set<string>, right: Set<string>) {
let count = 0;
for (const item of left) {
if (right.has(item)) {
count += 1;
}
}
return count;
}
function countProductSignals(phrase: string) {
return PRODUCT_SIGNAL_PATTERNS.filter((pattern) => pattern.test(phrase)).length;
}
function shouldKeepRelatedQuery(seedPhrase: string, relatedPhrase: string) {
const normalizedSeedPhrase = normalizePhrase(seedPhrase);
const normalizedRelatedPhrase = normalizePhrase(relatedPhrase);
if (!normalizedRelatedPhrase || normalizedRelatedPhrase === normalizedSeedPhrase) {
return false;
}
if (RELATED_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedRelatedPhrase))) {
return false;
}
const seedTokens = getMeaningfulTokenStems(normalizedSeedPhrase);
const relatedTokens = getMeaningfulTokenStems(normalizedRelatedPhrase);
const overlap = countIntersection(seedTokens, relatedTokens);
const productSignals = countProductSignals(normalizedRelatedPhrase);
const automationBridge =
AUTOMATION_SIGNAL_PATTERN.test(normalizedSeedPhrase) && AUTOMATION_SIGNAL_PATTERN.test(normalizedRelatedPhrase);
if (overlap >= 2) {
return true;
}
if (overlap >= 1 && productSignals >= 1) {
return true;
}
if (automationBridge && productSignals >= 1) {
return true;
}
return productSignals >= 2;
}
function mapJob(row: WordstatJobRow): NonNullable<WordstatEvidence["latestJob"]> {
return {
id: row.id,
status: row.status,
provider: row.provider,
mode: row.mode,
region: row.region,
requestedPhraseCount: Array.isArray(row.requested_phrases) ? row.requested_phrases.length : 0,
rawResultKey: row.raw_result_key,
resultCount: row.result_count,
errorMessage: row.error_message,
startedAt: toIsoDate(row.started_at),
completedAt: toIsoDate(row.completed_at),
createdAt: row.created_at.toISOString(),
updatedAt: row.updated_at.toISOString()
};
}
function getFrequencyBucket(frequencyGroup: string | null) {
const value = frequencyGroup?.toLowerCase();
if (!value) {
return "unknown";
}
if (["high", "vch", "вч"].includes(value)) {
return "high";
}
if (["mid", "medium", "sch", "сч"].includes(value)) {
return "mid";
}
if (["low", "nch", "нч"].includes(value)) {
return "low";
}
return "unknown";
}
function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]): WordstatEvidence {
const seedRows = results.filter((result) => result.source_phrase === result.phrase || !result.source_phrase);
const relatedCounts = new Map<string, number>();
const topResultByPhrase = new Map<string, WordstatResultRow>();
for (const result of results) {
if (result.source_phrase && result.source_phrase !== result.phrase) {
relatedCounts.set(result.source_phrase, (relatedCounts.get(result.source_phrase) ?? 0) + 1);
}
if (result.frequency !== null) {
const normalizedPhrase = result.normalized_phrase ?? normalizePhrase(result.phrase);
const currentResult = topResultByPhrase.get(normalizedPhrase);
if (!currentResult || (result.frequency ?? 0) > (currentResult.frequency ?? 0)) {
topResultByPhrase.set(normalizedPhrase, result);
}
}
}
return {
latestJob: job ? mapJob(job) : null,
seedResults: seedRows.map((result) => ({
seedId: result.seed_id,
phrase: result.phrase,
sourcePhrase: result.source_phrase,
frequency: result.frequency,
frequencyGroup: result.frequency_group,
region: result.region,
relatedCount: relatedCounts.get(result.phrase) ?? 0,
collectedAt: result.created_at.toISOString()
})),
topResults: Array.from(topResultByPhrase.values())
.sort((left, right) => (right.frequency ?? 0) - (left.frequency ?? 0))
.slice(0, 20)
.map((result) => ({
phrase: result.phrase,
sourcePhrase: result.source_phrase,
frequency: result.frequency,
frequencyGroup: result.frequency_group,
region: result.region
})),
summary: {
collectedSeedCount: seedRows.length,
exactFrequencySeedCount: seedRows.filter((result) => result.frequency !== null).length,
relatedOnlySeedCount: seedRows.filter(
(result) => result.frequency === null && (relatedCounts.get(result.phrase) ?? 0) > 0
).length,
noSignalSeedCount: seedRows.filter(
(result) => result.frequency === null && (relatedCounts.get(result.phrase) ?? 0) === 0
).length,
resultCount: results.length,
totalFrequency: results.reduce((sum, result) => sum + (result.frequency ?? 0), 0),
highFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "high").length,
midFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "mid").length,
lowFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "low").length
}
};
}
async function ensureSeedRows(projectId: string, analysis: SemanticAnalysisRun) {
const seedRows: SeedRow[] = [];
for (const seed of analysis.seedCandidates.slice(0, 80)) {
const inserted = await pool.query<SeedRow>(
`
insert into seeds (
project_id,
semantic_run_id,
phrase,
source,
selected,
reason,
confidence,
cluster_id,
cluster_title,
priority,
updated_at
)
values ($1, $2, $3, $4, true, $5, $6, $7, $8, $9, now())
on conflict do nothing
returning id, phrase, cluster_id, cluster_title, priority, source;
`,
[
projectId,
analysis.runId,
seed.phrase,
seed.source,
seed.reason,
seed.priority === "high" ? 0.85 : seed.priority === "medium" ? 0.65 : 0.45,
seed.clusterId,
seed.clusterTitle,
seed.priority
]
);
const row =
inserted.rows[0] ??
(
await pool.query<SeedRow>(
`
select id, phrase, cluster_id, cluster_title, priority, source
from seeds
where project_id = $1
and semantic_run_id = $2
and lower(phrase) = lower($3)
and coalesce(cluster_id, '') = coalesce($4, '')
order by created_at asc
limit 1;
`,
[projectId, analysis.runId, seed.phrase, seed.clusterId]
)
).rows[0];
if (row) {
seedRows.push(row);
}
}
return seedRows;
}
function toWordstatSeed(row: SeedRow): WordstatSeedInput {
return {
seedId: row.id,
phrase: row.phrase,
clusterId: row.cluster_id ?? "",
clusterTitle: row.cluster_title ?? "",
priority: row.priority ?? "medium",
source: row.source === "detected" ? "detected" : "ontology"
};
}
async function createJob(
projectId: string,
semanticRunId: string,
provider: string,
mode: string,
region: string,
seeds: WordstatSeedInput[]
) {
const result = await pool.query<WordstatJobRow>(
`
insert into wordstat_jobs (
project_id,
semantic_run_id,
status,
provider,
mode,
region,
requested_phrases,
started_at,
updated_at
)
values ($1, $2, 'running', $3, $4, $5, $6, now(), now())
returning
id,
project_id,
semantic_run_id,
status,
provider,
mode,
region,
requested_phrases,
raw_result_key,
result_count,
error_message,
started_at,
completed_at,
created_at,
updated_at;
`,
[projectId, semanticRunId, provider, mode, region, JSON.stringify(seeds.map((seed) => seed.phrase))]
);
return result.rows[0];
}
async function saveRawResult(projectId: string, semanticRunId: string, jobId: string, result: WordstatCollectResult) {
const rawResultKey = `projects/${projectId}/wordstat/${semanticRunId}/${jobId}/raw.json`;
await storageProvider.putObject({
key: rawResultKey,
body: JSON.stringify(result.raw, null, 2),
contentType: "application/json"
});
return rawResultKey;
}
async function saveResults(jobId: string, result: WordstatCollectResult) {
let resultCount = 0;
for (const seedResult of result.seedResults) {
await pool.query(
`
insert into wordstat_results (
job_id,
seed_id,
phrase,
source_phrase,
normalized_phrase,
frequency,
frequency_group,
region,
position,
raw
)
values ($1, $2, $3, $4, $5, $6, $7, $8, 0, $9);
`,
[
jobId,
seedResult.seedId,
seedResult.seedPhrase,
seedResult.seedPhrase,
normalizePhrase(seedResult.seedPhrase),
seedResult.frequency,
seedResult.frequencyGroup,
result.region,
{
source: "seed",
provider: result.provider,
mode: result.mode,
collectedAt: result.collectedAt,
raw: seedResult.raw
}
]
);
resultCount += 1;
const acceptedRelatedQueries = seedResult.relatedQueries.filter((relatedQuery) =>
shouldKeepRelatedQuery(seedResult.seedPhrase, relatedQuery.phrase)
);
for (const [index, relatedQuery] of acceptedRelatedQueries.entries()) {
await pool.query(
`
insert into wordstat_results (
job_id,
seed_id,
phrase,
source_phrase,
normalized_phrase,
frequency,
frequency_group,
region,
position,
raw
)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
`,
[
jobId,
seedResult.seedId,
relatedQuery.phrase,
seedResult.seedPhrase,
normalizePhrase(relatedQuery.phrase),
relatedQuery.frequency,
relatedQuery.frequencyGroup,
result.region,
index + 1,
{
source: "related",
provider: result.provider,
mode: result.mode,
collectedAt: result.collectedAt,
raw: relatedQuery.raw
}
]
);
resultCount += 1;
}
}
return resultCount;
}
async function updateJobDone(jobId: string, rawResultKey: string, resultCount: number) {
await pool.query(
`
update wordstat_jobs
set status = 'done',
raw_result_key = $2,
result_count = $3,
completed_at = now(),
updated_at = now()
where id = $1;
`,
[jobId, rawResultKey, resultCount]
);
}
async function updateJobFailed(jobId: string, error: unknown) {
const message = error instanceof Error ? error.message : "Не удалось собрать Wordstat evidence.";
await pool.query(
`
update wordstat_jobs
set status = 'failed',
error_message = $2,
completed_at = now(),
updated_at = now()
where id = $1;
`,
[jobId, message]
);
}
export async function collectWordstatEvidence(projectId: string, analysis: SemanticAnalysisRun) {
const provider = await createWordstatProvider();
const status = provider.getStatus();
if (!status.connected) {
throw new WordstatProviderUnavailableError(status.message);
}
const seedRows = await ensureSeedRows(projectId, analysis);
const seeds = seedRows.map(toWordstatSeed);
const region = "ru";
const job = await createJob(projectId, analysis.runId, status.provider, status.mode, region, seeds);
if (!job) {
throw new Error("Не удалось создать Wordstat job.");
}
try {
const result = await provider.collect({
projectId,
semanticRunId: analysis.runId,
region,
seeds
});
const rawResultKey = await saveRawResult(projectId, analysis.runId, job.id, result);
const resultCount = await saveResults(job.id, result);
await updateJobDone(job.id, rawResultKey, resultCount);
} catch (error) {
await updateJobFailed(job.id, error);
throw error;
}
return getLatestWordstatEvidence(projectId, analysis.runId);
}
export async function getLatestWordstatEvidence(projectId: string, semanticRunId: string | null): Promise<WordstatEvidence> {
if (!semanticRunId) {
return buildEvidence(null, []);
}
const jobResult = await pool.query<WordstatJobRow>(
`
select
id,
project_id,
semantic_run_id,
status,
provider,
mode,
region,
requested_phrases,
raw_result_key,
result_count,
error_message,
started_at,
completed_at,
created_at,
updated_at
from wordstat_jobs
where project_id = $1 and semantic_run_id = $2
order by created_at desc
limit 1;
`,
[projectId, semanticRunId]
);
const latestJob = jobResult.rows[0] ?? null;
if (!latestJob) {
return buildEvidence(null, []);
}
const results = await pool.query<WordstatResultRow>(
`
select
id,
job_id,
seed_id,
phrase,
source_phrase,
normalized_phrase,
frequency,
frequency_group,
region,
position,
raw,
created_at
from wordstat_results
where job_id = $1
order by position asc, phrase asc;
`,
[latestJob.id]
);
return buildEvidence(latestJob, results.rows);
}

View File

@ -0,0 +1,859 @@
import fs from "node:fs/promises";
import path from "node:path";
import { Router } from "express";
import type { Response } from "express";
import { pool } from "../db/client.js";
import { storageProvider } from "../storage/index.js";
import { annotatePreviewTargets, getPreviewTargetSeeds } from "../workspace/previewTargets.js";
import { getPreviewSnapshot } from "./snapshotCache.js";
type ProjectSourceRow = {
type: string;
config: Record<string, unknown>;
};
type PageSourceRow = {
source_path: string;
source_type: string;
source_config: Record<string, unknown>;
};
type ResolvedAsset = {
path: string;
body: Uint8Array;
contentType?: string;
};
type PreviewRewriteOptions = {
focusSelector?: string | null;
focusIndex?: number | null;
};
export const previewRouter = Router();
function getStringConfig(config: Record<string, unknown>, key: string) {
const value = config[key];
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function getSourceConfig(source: ProjectSourceRow | PageSourceRow) {
return "source_config" in source ? source.source_config : source.config;
}
function getSourceType(source: ProjectSourceRow | PageSourceRow) {
return "source_type" in source ? source.source_type : source.type;
}
function safeDecode(value: string) {
try {
return decodeURIComponent(value);
} catch {
return value;
}
}
function normalizePreviewPath(value: string) {
const withoutQuery = safeDecode(value).split("?")[0]?.split("#")[0] ?? "";
const normalized = path.posix.normalize(`/${withoutQuery.replace(/\\/g, "/")}`).replace(/^\/+/, "");
return normalized === "." ? "" : normalized;
}
function encodePathname(value: string) {
return value
.split("/")
.filter(Boolean)
.map((part) => encodeURIComponent(part))
.join("/");
}
function getMimeType(filePath: string) {
const extension = path.extname(filePath).toLowerCase();
const mimeTypes: Record<string, string> = {
".avif": "image/avif",
".css": "text/css; charset=utf-8",
".gif": "image/gif",
".htm": "text/html; charset=utf-8",
".html": "text/html; charset=utf-8",
".ico": "image/x-icon",
".jpeg": "image/jpeg",
".jpg": "image/jpeg",
".js": "text/javascript; charset=utf-8",
".json": "application/json; charset=utf-8",
".map": "application/json; charset=utf-8",
".mp4": "video/mp4",
".otf": "font/otf",
".png": "image/png",
".svg": "image/svg+xml; charset=utf-8",
".ttf": "font/ttf",
".txt": "text/plain; charset=utf-8",
".webm": "video/webm",
".webmanifest": "application/manifest+json; charset=utf-8",
".webp": "image/webp",
".woff": "font/woff",
".woff2": "font/woff2",
".xml": "application/xml; charset=utf-8"
};
return mimeTypes[extension] ?? "application/octet-stream";
}
function isHtml(filePath: string, contentType?: string) {
return /\.html?$/i.test(filePath) || Boolean(contentType?.includes("text/html"));
}
function isCss(filePath: string, contentType?: string) {
return /\.css$/i.test(filePath) || Boolean(contentType?.includes("text/css"));
}
function isTextLike(filePath: string, contentType?: string) {
return isHtml(filePath, contentType) || isCss(filePath, contentType) || /\.(js|json|svg|txt|xml|webmanifest)$/i.test(filePath);
}
function getSiteRoot(config: Record<string, unknown>, currentPath: string) {
const rootName = getStringConfig(config, "rootName");
if (rootName) {
const normalizedRoot = normalizePreviewPath(rootName);
if (normalizedRoot && (currentPath === normalizedRoot || currentPath.startsWith(`${normalizedRoot}/`))) {
return normalizedRoot;
}
}
const firstSegment = currentPath.split("/").filter(Boolean)[0] ?? "";
return firstSegment.includes(".") ? "" : firstSegment;
}
function buildCandidatePaths(requestedPath: string, config: Record<string, unknown>) {
const normalizedPath = normalizePreviewPath(requestedPath);
const rootName = normalizePreviewPath(getStringConfig(config, "rootName") ?? "");
const candidates = new Set<string>();
function addCandidate(candidate: string) {
const normalized = normalizePreviewPath(candidate);
if (!normalized) {
candidates.add("index.html");
return;
}
candidates.add(normalized);
if (normalized.endsWith("/")) {
candidates.add(`${normalized}index.html`);
}
}
addCandidate(normalizedPath);
if (rootName) {
if (!normalizedPath.startsWith(`${rootName}/`) && normalizedPath !== rootName) {
addCandidate(`${rootName}/${normalizedPath}`);
}
if (normalizedPath.startsWith(`${rootName}/`)) {
addCandidate(normalizedPath.slice(rootName.length + 1));
}
}
return [...candidates];
}
async function getLatestProjectSource(projectId: string) {
const result = await pool.query<ProjectSourceRow>(
`
select type, config
from project_sources
where project_id = $1
order by created_at desc
limit 1;
`,
[projectId]
);
return result.rows[0] ?? null;
}
async function getLatestPageSource(projectId: string, pageId: string) {
const result = await pool.query<PageSourceRow>(
`
select pv.source_path, ps.type as source_type, ps.config as source_config
from page_versions pv
join scan_versions sv on sv.id = pv.scan_version_id
join project_sources ps on ps.id = sv.source_id
where sv.project_id = $1
and pv.page_id = $2
and sv.status = 'done'
order by sv.completed_at desc nulls last, sv.created_at desc
limit 1;
`,
[projectId, pageId]
);
return result.rows[0] ?? null;
}
async function readBrowserFolderAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
const sourceConfig = getSourceConfig(source);
const importPrefix = getStringConfig(sourceConfig, "importPrefix");
if (!importPrefix) {
throw new Error("У проекта нет importPrefix.");
}
const normalizedPrefix = importPrefix.endsWith("/") ? importPrefix : `${importPrefix}/`;
for (const candidate of buildCandidatePaths(requestedPath, sourceConfig)) {
try {
const object = await storageProvider.getObject(`${normalizedPrefix}${candidate}`);
return {
path: candidate,
body: object.body,
contentType: object.contentType
};
} catch {
// Try the next deterministic candidate: rootName/path, path without rootName, etc.
}
}
return null;
}
async function readLocalFolderAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
const sourceConfig = getSourceConfig(source);
const sourceRoot = getStringConfig(sourceConfig, "path");
if (!sourceRoot) {
throw new Error("У проекта нет локального пути.");
}
const resolvedRoot = path.resolve(sourceRoot);
for (const candidate of buildCandidatePaths(requestedPath, sourceConfig)) {
const resolvedPath = path.resolve(resolvedRoot, candidate);
if (resolvedPath !== resolvedRoot && !resolvedPath.startsWith(`${resolvedRoot}${path.sep}`)) {
continue;
}
try {
return {
path: candidate,
body: await fs.readFile(resolvedPath),
contentType: getMimeType(candidate)
};
} catch {
// Try the next deterministic candidate.
}
}
return null;
}
async function readProjectAsset(source: ProjectSourceRow | PageSourceRow, requestedPath: string) {
if (getSourceType(source) === "browser_folder") {
return readBrowserFolderAsset(source, requestedPath);
}
if (getSourceType(source) === "local_folder") {
return readLocalFolderAsset(source, requestedPath);
}
throw new Error("Preview для этого типа источника пока не поддерживается.");
}
function sendNotFound(response: Response) {
response.status(404).type("text/plain").send("Preview asset not found.");
}
function splitUrlSuffix(value: string) {
const suffixIndex = value.search(/[?#]/);
if (suffixIndex === -1) {
return { pathname: value, suffix: "" };
}
return {
pathname: value.slice(0, suffixIndex),
suffix: value.slice(suffixIndex)
};
}
function shouldKeepOriginalUrl(value: string) {
const trimmed = value.trim();
return (
!trimmed ||
trimmed.startsWith("#") ||
trimmed.startsWith("//") ||
/^[a-z][a-z0-9+.-]*:/i.test(trimmed)
);
}
function buildPreviewSiteUrl(projectId: string, assetPath: string) {
return `/preview/projects/${encodeURIComponent(projectId)}/site/${encodePathname(assetPath)}`;
}
function buildPreviewDirectoryUrl(projectId: string, directoryPath: string) {
const normalizedDirectory = normalizePreviewPath(directoryPath);
const siteUrl = buildPreviewSiteUrl(projectId, normalizedDirectory);
return siteUrl.endsWith("/") ? siteUrl : `${siteUrl}/`;
}
function rewritePreviewUrl(value: string, projectId: string, siteRoot: string) {
const trimmed = value.trim();
if (shouldKeepOriginalUrl(trimmed) || !trimmed.startsWith("/") || trimmed.startsWith("//")) {
return value;
}
const { pathname, suffix } = splitUrlSuffix(trimmed);
const rootedPath = normalizePreviewPath(`${siteRoot}/${pathname.replace(/^\/+/, "")}`);
return `${buildPreviewSiteUrl(projectId, rootedPath)}${suffix}`;
}
function rewriteSrcset(value: string, projectId: string, siteRoot: string) {
return value
.split(",")
.map((entry) => {
const trimmedEntry = entry.trim();
if (!trimmedEntry) {
return entry;
}
const [url, ...descriptor] = trimmedEntry.split(/\s+/);
return [rewritePreviewUrl(url ?? "", projectId, siteRoot), ...descriptor].join(" ");
})
.join(", ");
}
function buildPreviewBridgeScript() {
return `
<script>
(() => {
const state = { workspaceKey: "", targets: [] };
const layerId = "seo-mode-preview-marker-layer";
let frame = 0;
let lastActivation = { id: "", at: 0 };
function ensureLayer() {
let layer = document.getElementById(layerId);
if (layer) {
return layer;
}
layer = document.createElement("div");
layer.id = layerId;
layer.setAttribute("aria-hidden", "true");
layer.style.position = "absolute";
layer.style.left = "0";
layer.style.top = "0";
layer.style.zIndex = "2147483647";
layer.style.pointerEvents = "none";
document.body.appendChild(layer);
return layer;
}
function scheduleMeasure() {
window.cancelAnimationFrame(frame);
frame = window.requestAnimationFrame(measure);
}
function syncLayerBounds(layer) {
const root = document.documentElement;
const width = Math.max(root.scrollWidth, document.body?.scrollWidth || 0, window.innerWidth);
const height = Math.max(root.scrollHeight, document.body?.scrollHeight || 0, window.innerHeight);
layer.style.width = width + "px";
layer.style.height = height + "px";
}
function intersects(rect) {
return rect.right > rect.left && rect.bottom > rect.top;
}
function intersectRects(rect, clipRect) {
return {
left: Math.max(rect.left, clipRect.left),
top: Math.max(rect.top, clipRect.top),
right: Math.min(rect.right, clipRect.right),
bottom: Math.min(rect.bottom, clipRect.bottom)
};
}
function getVisibleRect(element) {
let visibleRect = element.getBoundingClientRect();
if (visibleRect.width < 2 || visibleRect.height < 2) return null;
visibleRect = {
left: visibleRect.left,
top: visibleRect.top,
right: visibleRect.right,
bottom: visibleRect.bottom
};
visibleRect = intersectRects(visibleRect, {
left: 0,
top: 0,
right: window.innerWidth,
bottom: window.innerHeight
});
if (!intersects(visibleRect)) return null;
let parent = element.parentElement;
while (parent && parent !== document.body && parent !== document.documentElement) {
const parentStyle = window.getComputedStyle(parent);
const clipsContent = /(auto|scroll|hidden|clip)/.test(
[parentStyle.overflow, parentStyle.overflowX, parentStyle.overflowY].join(" ")
);
if (clipsContent) {
visibleRect = intersectRects(visibleRect, parent.getBoundingClientRect());
if (!intersects(visibleRect)) return null;
}
parent = parent.parentElement;
}
return visibleRect;
}
function getUsableRect(selector) {
if (!selector) return null;
const element = document.querySelector(selector);
if (!element) return null;
const style = window.getComputedStyle(element);
if (style.display === "none" || style.visibility === "hidden" || Number(style.opacity) === 0) return null;
return getVisibleRect(element);
}
function getHorizontalScroller() {
const candidates = Array.from(document.querySelectorAll("*"))
.filter((element) => element instanceof HTMLElement)
.filter((element) => element.scrollWidth > element.clientWidth + 8)
.map((element) => {
const rect = element.getBoundingClientRect();
const overflow = element.scrollWidth - element.clientWidth;
const area = Math.max(0, rect.width) * Math.max(0, rect.height);
return { element, overflow, area };
})
.filter((candidate) => candidate.area > 1000);
candidates.sort((first, second) => {
if (second.overflow !== first.overflow) return second.overflow - first.overflow;
return second.area - first.area;
});
return candidates[0]?.element ?? null;
}
function getPageStartPosition() {
const scroller = getHorizontalScroller();
if (scroller) {
const rect = scroller.getBoundingClientRect();
return {
left: rect.left + window.scrollX - scroller.scrollLeft + 14,
top: rect.top + window.scrollY + 14
};
}
return { left: 14, top: 14 };
}
function activateTarget(target, event) {
event.preventDefault();
event.stopPropagation();
if (typeof event.stopImmediatePropagation === "function") {
event.stopImmediatePropagation();
}
const now = Date.now();
if (lastActivation.id === target.id && now - lastActivation.at < 250) {
return;
}
lastActivation = { id: target.id, at: now };
window.parent?.postMessage({
type: "seo-mode:activate-target",
workspaceKey: state.workspaceKey,
targetId: target.id,
sectionId: target.sectionId || ""
}, "*");
}
function createMarker(target, left, top) {
const marker = document.createElement("button");
marker.type = "button";
marker.textContent = String(target.number);
marker.title = target.label || "Открыть поле правки";
marker.style.position = "absolute";
marker.style.left = left + "px";
marker.style.top = top + "px";
marker.style.width = "30px";
marker.style.height = "30px";
marker.style.display = "grid";
marker.style.placeItems = "center";
marker.style.padding = "0";
marker.style.color = "#fff";
marker.style.background = "#f27a1a";
marker.style.border = "2px solid rgba(255,255,255,.95)";
marker.style.borderRadius = "999px";
marker.style.boxShadow = "0 6px 16px rgba(242,122,26,.28)";
marker.style.cursor = "pointer";
marker.style.font = "700 12px/1 Inter, Arial, sans-serif";
marker.style.pointerEvents = "auto";
marker.style.touchAction = "manipulation";
marker.style.userSelect = "none";
marker.addEventListener("pointerdown", (event) => activateTarget(target, event));
marker.addEventListener("click", (event) => activateTarget(target, event));
return marker;
}
function measure() {
const layer = ensureLayer();
const root = document.documentElement;
const documentWidth = Math.max(root.scrollWidth, document.body?.scrollWidth || 0, window.innerWidth);
syncLayerBounds(layer);
layer.textContent = "";
const clusters = new Map();
for (const target of state.targets) {
try {
const exactSelectors = Array.isArray(target.selectors) && target.selectors.length > 0
? target.selectors
: [];
const prefersPageStart = target.preferPageStart || (target.pageStartFallback && exactSelectors.length === 0);
const selectors = prefersPageStart ? [] : (exactSelectors.length > 0 ? exactSelectors : [target.selector].filter(Boolean));
let placedExactMarker = false;
let placedFallbackMarker = false;
for (const selector of selectors) {
const rect = getUsableRect(selector);
if (!rect) continue;
placedExactMarker = true;
const baseLeft = Math.max(12, rect.left + window.scrollX + 10);
const baseTop = Math.max(12, rect.top + window.scrollY + 10);
const clusterKey = Math.round(baseLeft / 40) + ":" + Math.round(baseTop / 40);
const cluster = clusters.get(clusterKey) || [];
cluster.push({ target, baseLeft, baseTop });
clusters.set(clusterKey, cluster);
}
if (!prefersPageStart && !placedExactMarker && target.fallbackSelector) {
const rect = getUsableRect(target.fallbackSelector);
if (rect) {
placedFallbackMarker = true;
const baseLeft = Math.max(12, rect.left + window.scrollX + 10);
const baseTop = Math.max(12, rect.top + window.scrollY + 10);
const clusterKey = Math.round(baseLeft / 40) + ":" + Math.round(baseTop / 40);
const cluster = clusters.get(clusterKey) || [];
cluster.push({ target, baseLeft, baseTop });
clusters.set(clusterKey, cluster);
}
}
if (!placedExactMarker && !placedFallbackMarker && target.pageStartFallback) {
const pageStartPosition = getPageStartPosition();
const baseLeft = pageStartPosition.left;
const baseTop = pageStartPosition.top;
const clusterKey = "page-start";
const cluster = clusters.get(clusterKey) || [];
cluster.push({ target, baseLeft, baseTop, allowViewportOverflow: true });
clusters.set(clusterKey, cluster);
}
} catch {
// A broken selector should not break the preview bridge.
}
}
for (const cluster of clusters.values()) {
const markerSize = 30;
const gap = 6;
const maxColumns = 8;
const columns = Math.min(maxColumns, cluster.length);
const clusterWidth = columns * markerSize + Math.max(0, columns - 1) * gap;
const shouldClamp = !cluster.some((entry) => entry.allowViewportOverflow);
const startLeft = shouldClamp
? Math.min(
Math.max(12, cluster[0].baseLeft),
Math.max(12, documentWidth - clusterWidth - 12)
)
: cluster[0].baseLeft;
const startTop = Math.max(12, cluster[0].baseTop);
cluster.forEach((entry, index) => {
const column = index % columns;
const row = Math.floor(index / columns);
layer.appendChild(
createMarker(
entry.target,
startLeft + column * (markerSize + gap),
startTop + row * (markerSize + gap)
)
);
});
}
}
window.addEventListener("message", (event) => {
const data = event.data;
if (!data || data.type !== "seo-mode:measure-targets") {
return;
}
state.workspaceKey = data.workspaceKey || "";
state.targets = Array.isArray(data.targets) ? data.targets : [];
scheduleMeasure();
});
window.addEventListener("resize", scheduleMeasure);
window.addEventListener("scroll", scheduleMeasure, { passive: true });
document.addEventListener("scroll", scheduleMeasure, { capture: true, passive: true });
window.addEventListener("load", () => {
window.setTimeout(scheduleMeasure, 50);
window.setTimeout(scheduleMeasure, 500);
window.setTimeout(scheduleMeasure, 1500);
});
document.addEventListener("DOMContentLoaded", () => {
window.setTimeout(scheduleMeasure, 50);
window.setTimeout(scheduleMeasure, 500);
});
})();
</script>
`;
}
function buildPreviewFocusScript(selector: string | null | undefined, index: number | null | undefined = null) {
if (!selector) {
return "";
}
return `
<script>
(() => {
const selector = ${JSON.stringify(selector)};
const targetIndex = ${Number.isInteger(index) && index !== null ? index : 0};
const focusTarget = () => {
let target = null;
try {
const matches = Array.from(document.querySelectorAll(selector));
target = matches[Math.max(0, Math.min(targetIndex, matches.length - 1))] || null;
} catch {
target = null;
}
if (!target) return;
target.scrollIntoView({ block: "center", inline: "center" });
if (target instanceof HTMLElement) {
target.style.outline = "2px solid rgba(34, 197, 94, 0.55)";
target.style.outlineOffset = "8px";
}
};
window.addEventListener("load", () => {
window.setTimeout(focusTarget, 80);
window.setTimeout(focusTarget, 600);
window.setTimeout(focusTarget, 1400);
});
document.addEventListener("DOMContentLoaded", () => window.setTimeout(focusTarget, 80));
})();
</script>
`;
}
function getQueryString(value: unknown) {
const candidate = Array.isArray(value) ? value[0] : value;
if (typeof candidate !== "string") {
return null;
}
const trimmed = candidate.trim();
return trimmed.length > 0 && trimmed.length < 300 ? trimmed : null;
}
function getQueryInteger(value: unknown) {
const candidate = Array.isArray(value) ? value[0] : value;
const parsed = typeof candidate === "string" ? Number.parseInt(candidate, 10) : Number.NaN;
return Number.isInteger(parsed) && parsed >= 0 && parsed < 200 ? parsed : null;
}
function getQueryDimension(value: unknown, fallback: number) {
const candidate = Array.isArray(value) ? value[0] : value;
const parsed = typeof candidate === "string" ? Number.parseInt(candidate, 10) : Number.NaN;
return Number.isInteger(parsed) && parsed >= 320 && parsed <= 4096 ? parsed : fallback;
}
function rewriteHtml(
html: string,
projectId: string,
resolvedPath: string,
sourceConfig: Record<string, unknown>,
options: PreviewRewriteOptions = {}
) {
const siteRoot = getSiteRoot(sourceConfig, resolvedPath);
const currentDirectory = path.posix.dirname(resolvedPath);
const baseDirectory = currentDirectory === "." ? "" : `${currentDirectory}/`;
const baseHref = buildPreviewDirectoryUrl(projectId, baseDirectory);
const annotatedHtml = annotatePreviewTargets(html, getPreviewTargetSeeds(html));
const withoutBase = annotatedHtml.replace(/<base\b[^>]*>/gi, "");
const withRootedAttributes = withoutBase
.replace(/\s(src|href|poster|action)\s*=\s*(["'])([^"']*)\2/gi, (match, attribute: string, quote: string, value: string) => {
return ` ${attribute}=${quote}${rewritePreviewUrl(value, projectId, siteRoot)}${quote}`;
})
.replace(/\s(srcset)\s*=\s*(["'])([^"']*)\2/gi, (match, attribute: string, quote: string, value: string) => {
return ` ${attribute}=${quote}${rewriteSrcset(value, projectId, siteRoot)}${quote}`;
});
const baseTag = `<base href="${baseHref}">`;
const bridgeScript = buildPreviewBridgeScript();
const focusScript = buildPreviewFocusScript(options.focusSelector, options.focusIndex);
if (/<head\b[^>]*>/i.test(withRootedAttributes)) {
return withRootedAttributes
.replace(/<head\b[^>]*>/i, (headTag) => `${headTag}${baseTag}`)
.replace(/<\/body>/i, `${bridgeScript}${focusScript}</body>`);
}
return `<!doctype html><html><head>${baseTag}</head><body>${withRootedAttributes}${bridgeScript}${focusScript}</body></html>`;
}
function rewriteCss(css: string, projectId: string, resolvedPath: string, sourceConfig: Record<string, unknown>) {
const siteRoot = getSiteRoot(sourceConfig, resolvedPath);
return css
.replace(/url\(\s*(["']?)([^"')]+)\1\s*\)/gi, (_match, quote: string, value: string) => {
const rewritten = rewritePreviewUrl(value, projectId, siteRoot);
return `url(${quote}${rewritten}${quote})`;
})
.replace(/@import\s+(["'])([^"']+)\1/gi, (_match, quote: string, value: string) => {
return `@import ${quote}${rewritePreviewUrl(value, projectId, siteRoot)}${quote}`;
});
}
function textFromAsset(asset: ResolvedAsset) {
return new TextDecoder().decode(asset.body);
}
function sendAsset(response: Response, asset: ResolvedAsset, body: string | Uint8Array, contentType?: string) {
response.setHeader("Cache-Control", "no-store");
response.type(contentType ?? asset.contentType ?? getMimeType(asset.path));
response.send(typeof body === "string" ? body : Buffer.from(body));
}
previewRouter.get("/projects/:projectId/pages/:pageId", async (request, response) => {
try {
const page = await getLatestPageSource(request.params.projectId, request.params.pageId);
if (!page) {
sendNotFound(response);
return;
}
response.redirect(302, buildPreviewSiteUrl(request.params.projectId, page.source_path));
} catch (error) {
console.error(error);
response.status(500).type("text/plain").send("Preview page failed.");
}
});
previewRouter.get(/^\/projects\/([^/]+)\/snapshot\/?(.*)$/i, async (request, response) => {
try {
const params = request.params as Record<string, string | undefined>;
const projectId = params[0] ?? "";
const rawAssetPath = params[1] ?? "";
const cacheVersion = getQueryString(request.query.cacheVersion) ?? getQueryString(request.query.scanId) ?? "latest";
const focusSelector = getQueryString(request.query.focus);
const focusIndex = getQueryInteger(request.query.focusIndex);
const viewportWidth = getQueryDimension(request.query.viewportWidth, 1920);
const viewportHeight = getQueryDimension(request.query.viewportHeight, 1080);
const origin = `${request.protocol}://${request.get("host")}`;
const snapshot = await getPreviewSnapshot({
focusIndex,
focusSelector,
origin,
projectId,
cacheVersion,
sourcePath: rawAssetPath || "index.html",
viewportHeight,
viewportWidth
});
response
.status(200)
.setHeader("Cache-Control", "public, max-age=31536000, immutable")
.setHeader("X-SEO-Mode-Preview-Cache", snapshot.status)
.type("image/png")
.send(Buffer.from(snapshot.body));
} catch (error) {
console.error(error);
response.status(503).type("text/plain").send("Preview snapshot failed.");
}
});
previewRouter.get(/^\/projects\/([^/]+)\/site\/?(.*)$/i, async (request, response) => {
try {
const params = request.params as Record<string, string | undefined>;
const projectId = params[0] ?? "";
const rawAssetPath = params[1] ?? "";
const focusSelector = getQueryString(request.query.focus);
const focusIndex = getQueryInteger(request.query.focusIndex);
const source = await getLatestProjectSource(projectId);
if (!source) {
sendNotFound(response);
return;
}
const asset = await readProjectAsset(source, rawAssetPath || "index.html");
if (!asset) {
sendNotFound(response);
return;
}
const contentType = asset.contentType ?? getMimeType(asset.path);
if (isHtml(asset.path, contentType)) {
sendAsset(
response,
asset,
rewriteHtml(textFromAsset(asset), projectId, asset.path, source.config, { focusIndex, focusSelector }),
"text/html; charset=utf-8"
);
return;
}
if (isCss(asset.path, contentType)) {
sendAsset(response, asset, rewriteCss(textFromAsset(asset), projectId, asset.path, source.config), "text/css; charset=utf-8");
return;
}
sendAsset(response, asset, isTextLike(asset.path, contentType) ? textFromAsset(asset) : asset.body, contentType);
} catch (error) {
console.error(error);
response.status(500).type("text/plain").send("Preview asset failed.");
}
});

View File

@ -0,0 +1,189 @@
import crypto from "node:crypto";
import { chromium, type Browser } from "playwright-core";
import { env } from "../config/env.js";
import { storageProvider } from "../storage/index.js";
type SnapshotRequest = {
cacheVersion: string;
focusIndex: number | null;
focusSelector: string | null;
origin: string;
projectId: string;
sourcePath: string;
viewportHeight: number;
viewportWidth: number;
};
type QueueTask<T> = {
reject: (error: unknown) => void;
resolve: (value: T) => void;
run: () => Promise<T>;
};
const inFlightSnapshots = new Map<string, Promise<Uint8Array>>();
const queue: QueueTask<Uint8Array>[] = [];
let activeTasks = 0;
let browserPromise: Promise<Browser> | null = null;
function encodePathname(value: string) {
return value
.split("/")
.filter(Boolean)
.map((part) => encodeURIComponent(part))
.join("/");
}
function getSnapshotCacheKey(input: SnapshotRequest) {
const hash = crypto
.createHash("sha256")
.update(
JSON.stringify({
focusIndex: input.focusIndex,
focusSelector: input.focusSelector,
sourcePath: input.sourcePath,
viewportHeight: input.viewportHeight,
viewportWidth: input.viewportWidth
})
)
.digest("hex")
.slice(0, 32);
return `preview-cache/${input.projectId}/${input.cacheVersion}/${hash}.png`;
}
function buildPreviewUrl(input: SnapshotRequest) {
const url = new URL(`/preview/projects/${encodeURIComponent(input.projectId)}/site/${encodePathname(input.sourcePath)}`, input.origin);
if (input.focusSelector) {
url.searchParams.set("focus", input.focusSelector);
}
if (input.focusIndex != null) {
url.searchParams.set("focusIndex", String(input.focusIndex));
}
return url.toString();
}
async function getBrowser() {
if (!browserPromise) {
browserPromise = chromium.launch({
executablePath: env.preview.chromePath,
headless: true,
args: ["--disable-dev-shm-usage", "--no-sandbox"]
});
}
return browserPromise;
}
function pumpQueue() {
while (activeTasks < env.preview.snapshotConcurrency && queue.length > 0) {
const task = queue.shift();
if (!task) {
return;
}
activeTasks += 1;
void task
.run()
.then(task.resolve, task.reject)
.finally(() => {
activeTasks -= 1;
pumpQueue();
});
}
}
function enqueueSnapshot(run: () => Promise<Uint8Array>) {
return new Promise<Uint8Array>((resolve, reject) => {
queue.push({ reject, resolve, run });
pumpQueue();
});
}
async function readCachedSnapshot(key: string) {
try {
const cached = await storageProvider.getObject(key);
if (cached.body.byteLength > 0) {
return cached.body;
}
} catch {
return null;
}
return null;
}
async function generateSnapshot(input: SnapshotRequest, cacheKey: string) {
const browser = await getBrowser();
const page = await browser.newPage({
deviceScaleFactor: 1,
viewport: {
height: input.viewportHeight,
width: input.viewportWidth
}
});
try {
page.setDefaultTimeout(18_000);
await page.goto(buildPreviewUrl(input), { timeout: 25_000, waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle", { timeout: 8_000 }).catch(() => undefined);
await page.waitForTimeout(env.preview.snapshotWaitMs);
const screenshot = await page.screenshot({
animations: "disabled",
fullPage: false,
type: "png"
});
const body = new Uint8Array(screenshot);
await storageProvider.putObject({
body,
contentType: "image/png",
key: cacheKey
});
return body;
} finally {
await page.close().catch(() => undefined);
}
}
export async function getPreviewSnapshot(input: SnapshotRequest) {
const cacheKey = getSnapshotCacheKey(input);
const cached = await readCachedSnapshot(cacheKey);
if (cached) {
return {
body: cached,
cacheKey,
status: "hit" as const
};
}
const existing = inFlightSnapshots.get(cacheKey);
if (existing) {
return {
body: await existing,
cacheKey,
status: "joined" as const
};
}
const generated = enqueueSnapshot(() => generateSnapshot(input, cacheKey));
inFlightSnapshots.set(cacheKey, generated);
try {
return {
body: await generated,
cacheKey,
status: "miss" as const
};
} finally {
inFlightSnapshots.delete(cacheKey);
}
}

View File

@ -0,0 +1,43 @@
import { constants } from "node:fs";
import { access, stat } from "node:fs/promises";
import path from "node:path";
export class LocalFolderValidationError extends Error {
constructor(message: string) {
super(message);
this.name = "LocalFolderValidationError";
}
}
export async function validateLocalFolderPath(inputPath: string) {
const trimmedPath = inputPath.trim();
if (!trimmedPath) {
throw new LocalFolderValidationError("Укажи путь к локальной папке проекта.");
}
if (!path.isAbsolute(trimmedPath)) {
throw new LocalFolderValidationError("Путь к локальной папке должен быть абсолютным.");
}
const resolvedPath = path.resolve(trimmedPath);
let stats;
try {
stats = await stat(resolvedPath);
} catch {
throw new LocalFolderValidationError("Такой папки не существует или сервер не может её прочитать.");
}
if (!stats.isDirectory()) {
throw new LocalFolderValidationError("Указанный путь должен вести именно к папке, а не к файлу.");
}
try {
await access(resolvedPath, constants.R_OK);
} catch {
throw new LocalFolderValidationError("У сервера нет прав на чтение этой папки.");
}
return resolvedPath;
}

View File

@ -0,0 +1,712 @@
import type { PoolClient } from "pg";
import crypto from "node:crypto";
import path from "node:path";
import { pool } from "../db/client.js";
import { storageProvider } from "../storage/index.js";
import { validateLocalFolderPath } from "./localFolder.js";
type ProjectRow = {
id: string;
name: string;
slug: string;
created_at: Date;
updated_at: Date;
source_id: string | null;
source_type: string | null;
source_config: Record<string, unknown> | null;
source_created_at: Date | null;
source_updated_at: Date | null;
};
type ProjectRecord = {
id: string;
name: string;
slug: string;
created_at: Date;
updated_at: Date;
};
type SourceRecord = {
id: string;
type: string;
config: Record<string, unknown>;
created_at: Date;
updated_at: Date;
};
type SourceConfigRow = {
config: Record<string, unknown>;
};
type ProjectSourceCopyRow = {
id: string;
name: string;
source_id: string;
source_type: string;
source_config: Record<string, unknown>;
};
export type ProjectSummary = {
id: string;
name: string;
slug: string;
createdAt: string;
updatedAt: string;
source: {
id: string;
type: string;
path: string | null;
config: Record<string, unknown>;
createdAt: string;
updatedAt: string;
} | null;
};
export type CreateProjectInput = {
name?: string;
sourcePath: string;
};
export type BrowserFolderFileInput = {
relativePath: string;
contentBase64: string;
contentType?: string;
size: number;
lastModified?: number;
};
export type CreateBrowserFolderProjectInput = {
name?: string;
rootName: string;
files: BrowserFolderFileInput[];
};
export class ProjectNameConflictError extends Error {
constructor(name: string) {
super(`Проект с названием «${name}» уже есть.`);
this.name = "ProjectNameConflictError";
}
}
export class ProjectCopyError extends Error {
constructor(message: string) {
super(message);
this.name = "ProjectCopyError";
}
}
const CYRILLIC_TO_LATIN: Record<string, string> = {
а: "a",
б: "b",
в: "v",
г: "g",
д: "d",
е: "e",
ё: "e",
ж: "zh",
з: "z",
и: "i",
й: "y",
к: "k",
л: "l",
м: "m",
н: "n",
о: "o",
п: "p",
р: "r",
с: "s",
т: "t",
у: "u",
ф: "f",
х: "h",
ц: "ts",
ч: "ch",
ш: "sh",
щ: "sch",
ъ: "",
ы: "y",
ь: "",
э: "e",
ю: "yu",
я: "ya"
};
const MAX_PROJECT_NAME_LENGTH = 120;
function toIsoDate(value: Date) {
return value.toISOString();
}
function getSourcePath(config: Record<string, unknown>) {
return typeof config.path === "string" ? config.path : null;
}
function sanitizeObjectPath(value: string) {
const normalizedPath = value.replace(/\\/g, "/").replace(/^\/+/, "");
const segments = normalizedPath.split("/").filter(Boolean);
if (segments.length === 0) {
throw new Error("Файл без относительного пути нельзя импортировать.");
}
if (segments.some((segment) => segment === "." || segment === "..")) {
throw new Error("В импорте найден небезопасный путь файла.");
}
return segments.join("/");
}
function mapProjectRow(row: ProjectRow): ProjectSummary {
const sourceConfig = row.source_config ?? {};
return {
id: row.id,
name: row.name,
slug: row.slug,
createdAt: toIsoDate(row.created_at),
updatedAt: toIsoDate(row.updated_at),
source:
row.source_id && row.source_type && row.source_created_at && row.source_updated_at
? {
id: row.source_id,
type: row.source_type,
path: getSourcePath(sourceConfig),
config: sourceConfig,
createdAt: toIsoDate(row.source_created_at),
updatedAt: toIsoDate(row.source_updated_at)
}
: null
};
}
function mapProjectRecord(project: ProjectRecord, source: SourceRecord): ProjectSummary {
return {
id: project.id,
name: project.name,
slug: project.slug,
createdAt: toIsoDate(project.created_at),
updatedAt: toIsoDate(project.updated_at),
source: {
id: source.id,
type: source.type,
path: getSourcePath(source.config),
config: source.config,
createdAt: toIsoDate(source.created_at),
updatedAt: toIsoDate(source.updated_at)
}
};
}
async function selectProjectById(client: PoolClient, projectId: string) {
const result = await client.query<ProjectRow>(
`
select
p.id,
p.name,
p.slug,
p.created_at,
p.updated_at,
ps.id as source_id,
ps.type as source_type,
ps.config as source_config,
ps.created_at as source_created_at,
ps.updated_at as source_updated_at
from projects p
left join lateral (
select id, type, config, created_at, updated_at
from project_sources
where project_id = p.id
order by created_at asc
limit 1
) ps on true
where p.id = $1;
`,
[projectId]
);
return result.rows[0] ? mapProjectRow(result.rows[0]) : null;
}
async function selectProjectSourceForCopy(client: PoolClient, projectId: string) {
const result = await client.query<ProjectSourceCopyRow>(
`
select
p.id,
p.name,
ps.id as source_id,
ps.type as source_type,
ps.config as source_config
from projects p
join lateral (
select id, type, config
from project_sources
where project_id = p.id
order by created_at asc
limit 1
) ps on true
where p.id = $1;
`,
[projectId]
);
return result.rows[0] ?? null;
}
async function insertProjectWithSource(
client: PoolClient,
input: {
name: string;
sourceType: "local_folder" | "browser_folder";
sourceConfig: Record<string, unknown>;
}
) {
const slug = await buildUniqueSlug(client, input.name);
const projectResult = await client.query<ProjectRecord>(
`
insert into projects (name, slug)
values ($1, $2)
returning id, name, slug, created_at, updated_at;
`,
[input.name, slug]
);
const project = projectResult.rows[0];
if (!project) {
throw new Error("Не удалось создать проект.");
}
const sourceResult = await client.query<SourceRecord>(
`
insert into project_sources (project_id, type, config)
values ($1, $2, $3)
returning id, type, config, created_at, updated_at;
`,
[project.id, input.sourceType, input.sourceConfig]
);
const source = sourceResult.rows[0];
if (!source) {
throw new Error("Не удалось сохранить источник проекта.");
}
return { project, source };
}
function transliterate(value: string) {
return value
.toLowerCase()
.split("")
.map((char) => CYRILLIC_TO_LATIN[char] ?? char)
.join("");
}
function slugifyName(name: string) {
const slug = transliterate(name)
.normalize("NFKD")
.replace(/[\u0300-\u036f]/g, "")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 64);
return slug || "project";
}
async function slugExists(client: PoolClient, slug: string, excludeProjectId?: string) {
const result = excludeProjectId
? await client.query<{ exists: boolean }>(
"select exists(select 1 from projects where slug = $1 and id <> $2) as exists",
[slug, excludeProjectId]
)
: await client.query<{ exists: boolean }>(
"select exists(select 1 from projects where slug = $1) as exists",
[slug]
);
return result.rows[0]?.exists ?? false;
}
async function projectNameExists(client: PoolClient, name: string, excludeProjectId?: string) {
const result = excludeProjectId
? await client.query<{ exists: boolean }>(
"select exists(select 1 from projects where lower(name) = lower($1) and id <> $2) as exists",
[name, excludeProjectId]
)
: await client.query<{ exists: boolean }>(
"select exists(select 1 from projects where lower(name) = lower($1)) as exists",
[name]
);
return result.rows[0]?.exists ?? false;
}
async function assertProjectNameAvailable(client: PoolClient, name: string, excludeProjectId?: string) {
if (await projectNameExists(client, name, excludeProjectId)) {
throw new ProjectNameConflictError(name);
}
}
async function buildUniqueSlug(client: PoolClient, name: string, excludeProjectId?: string) {
const baseSlug = slugifyName(name);
let suffix = 1;
while (suffix < 1000) {
const suffixText = suffix === 1 ? "" : `-${suffix}`;
const candidate = `${baseSlug.slice(0, 64 - suffixText.length)}${suffixText}`;
if (!(await slugExists(client, candidate, excludeProjectId))) {
return candidate;
}
suffix += 1;
}
return `${baseSlug.slice(0, 50)}-${Date.now()}`;
}
function getCopyBaseName(originalName: string) {
let baseName = originalName.trim() || "project";
let previousBaseName = "";
while (baseName && baseName !== previousBaseName) {
previousBaseName = baseName;
baseName = baseName
.replace(/^(?:(?:\d{2}|[1-9]\d{0,2})\s*-\s*копия\s*-\s*|Копия(?:\s+\d+)?\s*-\s*|[1-9]\d{0,2}\s*-\s*)/iu, "")
.trim();
}
return baseName || "project";
}
function buildCopyNameCandidate(originalName: string, copyIndex: number) {
const prefix = `${String(copyIndex).padStart(2, "0")} - копия - `;
const normalizedOriginalName = getCopyBaseName(originalName);
const maxOriginalNameLength = Math.max(1, MAX_PROJECT_NAME_LENGTH - prefix.length);
return `${prefix}${normalizedOriginalName.slice(0, maxOriginalNameLength)}`;
}
async function buildUniqueCopyName(client: PoolClient, originalName: string) {
let copyIndex = 1;
while (copyIndex < 1000) {
const candidate = buildCopyNameCandidate(originalName, copyIndex);
if (!(await projectNameExists(client, candidate))) {
return candidate;
}
copyIndex += 1;
}
return buildCopyNameCandidate(`${crypto.randomUUID().slice(0, 8)}-${originalName}`, 1);
}
export async function listProjects() {
const result = await pool.query<ProjectRow>(`
select
p.id,
p.name,
p.slug,
p.created_at,
p.updated_at,
ps.id as source_id,
ps.type as source_type,
ps.config as source_config,
ps.created_at as source_created_at,
ps.updated_at as source_updated_at
from projects p
left join lateral (
select id, type, config, created_at, updated_at
from project_sources
where project_id = p.id
order by created_at asc
limit 1
) ps on true
order by p.created_at desc;
`);
return result.rows.map(mapProjectRow);
}
export async function createProject(input: CreateProjectInput) {
const resolvedSourcePath = await validateLocalFolderPath(input.sourcePath);
const name = input.name?.trim() || path.basename(resolvedSourcePath) || "project";
const client = await pool.connect();
try {
await client.query("begin");
await assertProjectNameAvailable(client, name);
const { project, source } = await insertProjectWithSource(client, {
name,
sourceType: "local_folder",
sourceConfig: {
path: resolvedSourcePath,
originalPath: input.sourcePath.trim()
}
});
await client.query("commit");
return mapProjectRecord(project, source);
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
}
export async function createBrowserFolderProject(input: CreateBrowserFolderProjectInput) {
const rootName = input.rootName.trim() || "browser-folder";
const name = input.name?.trim() || rootName;
const files = input.files.map((file) => ({
...file,
relativePath: sanitizeObjectPath(file.relativePath)
}));
const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
const client = await pool.connect();
if (files.length === 0) {
throw new Error("В выбранной папке нет файлов для импорта.");
}
try {
await client.query("begin");
await assertProjectNameAvailable(client, name);
const importPrefixSeed = crypto.randomUUID();
const importPrefix = `browser-folder-imports/${importPrefixSeed}`;
const { project, source } = await insertProjectWithSource(client, {
name,
sourceType: "browser_folder",
sourceConfig: {
rootName,
importPrefix,
fileCount: files.length,
totalBytes,
importedAt: new Date().toISOString(),
sampleFiles: files.slice(0, 20).map((file) => file.relativePath)
}
});
await storageProvider.ensureReady();
for (const file of files) {
await storageProvider.putObject({
key: `${importPrefix}/${file.relativePath}`,
body: Buffer.from(file.contentBase64, "base64"),
contentType: file.contentType || "application/octet-stream"
});
}
await storageProvider.putObject({
key: `${importPrefix}/manifest.json`,
body: JSON.stringify(
{
projectId: project.id,
sourceId: source.id,
rootName,
fileCount: files.length,
totalBytes,
files: files.map((file) => ({
relativePath: file.relativePath,
contentType: file.contentType,
size: file.size,
lastModified: file.lastModified
}))
},
null,
2
),
contentType: "application/json"
});
await client.query("commit");
return mapProjectRecord(project, source);
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
}
export async function updateProjectName(projectId: string, name: string) {
const nextName = name.trim();
const client = await pool.connect();
if (!nextName) {
throw new Error("Название проекта обязательно.");
}
try {
await client.query("begin");
await assertProjectNameAvailable(client, nextName, projectId);
const nextSlug = await buildUniqueSlug(client, nextName, projectId);
const updateResult = await client.query(
`
update projects
set name = $1, slug = $2, updated_at = now()
where id = $3
returning id;
`,
[nextName, nextSlug, projectId]
);
if (updateResult.rowCount === 0) {
await client.query("rollback");
return null;
}
const project = await selectProjectById(client, projectId);
await client.query("commit");
return project;
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
}
export async function copyProject(projectId: string) {
const client = await pool.connect();
let copiedImportPrefix: string | null = null;
try {
await client.query("begin");
const sourceProject = await selectProjectSourceForCopy(client, projectId);
if (!sourceProject) {
await client.query("rollback");
return null;
}
const copiedAt = new Date().toISOString();
const sourceConfig: Record<string, unknown> = {
...sourceProject.source_config,
copiedFromProjectId: sourceProject.id,
copiedFromSourceId: sourceProject.source_id,
copiedAt
};
let sourceType: "local_folder" | "browser_folder";
let sourceImportPrefix: string | null = null;
if (sourceProject.source_type === "local_folder") {
sourceType = "local_folder";
} else if (sourceProject.source_type === "browser_folder") {
const importPrefix = sourceProject.source_config.importPrefix;
if (typeof importPrefix !== "string" || importPrefix.length === 0) {
throw new ProjectCopyError("У исходного проекта нет корректного prefix импорта.");
}
sourceType = "browser_folder";
sourceImportPrefix = importPrefix;
copiedImportPrefix = `browser-folder-imports/${crypto.randomUUID()}`;
sourceConfig.importPrefix = copiedImportPrefix;
sourceConfig.copiedFromImportPrefix = sourceImportPrefix;
} else {
throw new ProjectCopyError("Копирование этого типа источника пока не поддерживается.");
}
const copyName = await buildUniqueCopyName(client, sourceProject.name);
const { project, source } = await insertProjectWithSource(client, {
name: copyName,
sourceType,
sourceConfig
});
if (sourceType === "browser_folder" && sourceImportPrefix && copiedImportPrefix) {
const copiedObjects = await storageProvider.copyPrefix(sourceImportPrefix, copiedImportPrefix);
if (copiedObjects === 0) {
throw new ProjectCopyError("В хранилище не найдены файлы исходного проекта.");
}
await storageProvider.putObject({
key: `${copiedImportPrefix}/manifest.json`,
body: JSON.stringify(
{
projectId: project.id,
sourceId: source.id,
copiedFromProjectId: sourceProject.id,
copiedFromSourceId: sourceProject.source_id,
copiedFromImportPrefix: sourceImportPrefix,
copiedAt,
rootName: sourceConfig.rootName,
fileCount: sourceConfig.fileCount,
totalBytes: sourceConfig.totalBytes,
sampleFiles: sourceConfig.sampleFiles
},
null,
2
),
contentType: "application/json"
});
}
await client.query("commit");
return mapProjectRecord(project, source);
} catch (error) {
await client.query("rollback");
if (copiedImportPrefix) {
try {
await storageProvider.deletePrefix(copiedImportPrefix);
} catch (cleanupError) {
console.warn(`Не удалось подчистить файлы неудачной копии по prefix ${copiedImportPrefix}`, cleanupError);
}
}
throw error;
} finally {
client.release();
}
}
export async function deleteProject(projectId: string) {
const client = await pool.connect();
try {
const sourceConfigs = await client.query<SourceConfigRow>(
`
select config
from project_sources
where project_id = $1 and type = 'browser_folder';
`,
[projectId]
);
const importPrefixes = sourceConfigs.rows
.map((row) => row.config.importPrefix)
.filter((value): value is string => typeof value === "string" && value.length > 0);
await client.query("begin");
const deleteResult = await client.query("delete from projects where id = $1", [projectId]);
if (deleteResult.rowCount === 0) {
await client.query("rollback");
return false;
}
await client.query("commit");
for (const prefix of importPrefixes) {
try {
await storageProvider.deletePrefix(prefix);
} catch (error) {
console.warn(`Не удалось удалить файлы проекта из хранилища по prefix ${prefix}`, error);
}
}
return true;
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
}

View File

@ -0,0 +1,599 @@
import { Router } from "express";
import type { Response } from "express";
import crypto from "node:crypto";
import { z } from "zod";
import {
getLatestProjectScan,
ProjectScanError,
scanProject,
updateProjectPageSelection
} from "../scans/projectScanner.js";
import { LocalFolderValidationError } from "./localFolder.js";
import {
copyProject,
createBrowserFolderProject,
createProject,
deleteProject,
listProjects,
ProjectCopyError,
ProjectNameConflictError,
updateProjectName
} from "./projectRepository.js";
import {
getOrCreatePageWorkspace,
PageWorkspaceError,
resetPageWorkspace,
savePageWorkspace
} from "../workspace/pageWorkspace.js";
import {
getLatestSemanticAnalysis,
getSemanticAnalysisExport,
runSemanticAnalysis,
SemanticAnalysisError
} from "../analysis/semanticAnalysis.js";
import { getMarketEnrichmentContract, runMarketEnrichment } from "../market/marketEnrichment.js";
import { storageProvider } from "../storage/index.js";
export const projectsRouter = Router();
const createProjectSchema = z.object({
name: z.string().trim().max(120, "Название слишком длинное.").optional().default(""),
sourcePath: z.string().trim().min(1, "Путь к локальной папке обязателен.")
});
const browserFolderFileSchema = z.object({
relativePath: z.string().trim().min(1),
contentBase64: z.string().min(1),
contentType: z.string().optional(),
size: z.number().int().nonnegative(),
lastModified: z.number().optional()
});
const importBrowserFolderSchema = z.object({
name: z.string().trim().max(120, "Название слишком длинное.").optional().default(""),
rootName: z.string().trim().min(1, "Не удалось определить имя папки."),
files: z.array(browserFolderFileSchema).min(1, "В выбранной папке нет файлов для импорта.")
});
const projectIdSchema = z.string().uuid("Некорректный id проекта.");
const updateProjectNameSchema = z.object({
name: z.string().trim().min(1, "Название проекта обязательно.").max(120, "Название слишком длинное.")
});
const updatePageSelectionSchema = z.object({
pageIds: z.array(z.string().uuid("Некорректный id страницы.")).min(1, "Нужно выбрать хотя бы одну страницу."),
selected: z.boolean()
});
const updatePageWorkspaceSchema = z.object({
workingText: z.string().max(1_000_000, "Рабочий текст слишком большой для текущего MVP.")
});
const semanticExportFormatSchema = z.enum(["csv", "json", "markdown"]).default("json");
const scopeLayoutKeySchema = z.string().trim().min(1).max(240);
const scopeLayoutSchema = z.object({
version: z.number().int().positive(),
viewport: z
.object({
x: z.number().finite(),
y: z.number().finite(),
zoom: z.number().finite().positive()
})
.optional(),
nodes: z.record(
z.string(),
z.object({
position: z.object({
x: z.number().finite(),
y: z.number().finite()
}),
width: z.number().finite().positive().optional(),
height: z.number().finite().positive().optional()
})
)
});
const MAX_BROWSER_FOLDER_FILES = 1500;
const MAX_BROWSER_FOLDER_BYTES = 100 * 1024 * 1024;
function sendError(response: Response, status: number, message: string) {
response.status(status).json({
error: {
message
}
});
}
function getScopeLayoutObjectKey(projectId: string, layoutKey: string) {
const digest = crypto.createHash("sha256").update(layoutKey).digest("hex").slice(0, 40);
return `project-scope-layouts/${projectId}/${digest}.json`;
}
projectsRouter.get("/", async (_request, response) => {
try {
response.json({
projects: await listProjects()
});
} catch (error) {
console.error(error);
sendError(response, 500, "Не удалось загрузить проекты.");
}
});
projectsRouter.post("/", async (request, response) => {
try {
const input = createProjectSchema.parse(request.body);
const project = await createProject(input);
response.status(201).json({
project
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь данные проекта.");
return;
}
if (error instanceof LocalFolderValidationError) {
sendError(response, 400, error.message);
return;
}
if (error instanceof ProjectNameConflictError) {
sendError(response, 409, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось создать проект.");
}
});
projectsRouter.post("/import-folder", async (request, response) => {
try {
const input = importBrowserFolderSchema.parse(request.body);
const totalBytes = input.files.reduce((sum, file) => sum + file.size, 0);
if (input.files.length > MAX_BROWSER_FOLDER_FILES) {
sendError(response, 400, `Слишком много файлов: максимум ${MAX_BROWSER_FOLDER_FILES}.`);
return;
}
if (totalBytes > MAX_BROWSER_FOLDER_BYTES) {
sendError(response, 400, "Папка слишком большая для первого импорта: максимум 100 МБ.");
return;
}
const project = await createBrowserFolderProject(input);
response.status(201).json({
project
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь данные импорта папки.");
return;
}
if (error instanceof ProjectNameConflictError) {
sendError(response, 409, error.message);
return;
}
if (error instanceof ProjectCopyError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось импортировать папку.");
}
});
projectsRouter.post("/:projectId/copy", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const project = await copyProject(projectId);
if (!project) {
sendError(response, 404, "Проект не найден.");
return;
}
response.status(201).json({
project
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
if (error instanceof ProjectNameConflictError) {
sendError(response, 409, error.message);
return;
}
if (error instanceof Error && error.message) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось скопировать проект.");
}
});
projectsRouter.get("/:projectId/scans/latest", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
response.json({
scan: await getLatestProjectScan(projectId)
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось загрузить последний скан проекта.");
}
});
projectsRouter.post("/:projectId/scans", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const scan = await scanProject(projectId);
if (!scan) {
sendError(response, 404, "Проект не найден.");
return;
}
response.status(201).json({
scan
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
if (error instanceof ProjectScanError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось просканировать проект.");
}
});
projectsRouter.get("/:projectId/analysis/semantic/latest", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
response.json({
analysis: await getLatestSemanticAnalysis(projectId)
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось загрузить semantic analysis проекта.");
}
});
projectsRouter.post("/:projectId/analysis/semantic", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const analysis = await runSemanticAnalysis(projectId);
if (!analysis) {
sendError(response, 404, "Сначала нужен успешный скан проекта.");
return;
}
response.status(201).json({
analysis
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
if (error instanceof SemanticAnalysisError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось выполнить semantic analysis проекта.");
}
});
projectsRouter.get("/:projectId/analysis/semantic/export", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const format = semanticExportFormatSchema.parse(request.query.format);
const exportedAnalysis = await getSemanticAnalysisExport(projectId, format);
if (!exportedAnalysis) {
sendError(response, 404, "Сначала нужен semantic analysis проекта.");
return;
}
response
.status(200)
.setHeader("Content-Type", exportedAnalysis.contentType)
.setHeader("Content-Disposition", `attachment; filename="${exportedAnalysis.filename}"`)
.send(exportedAnalysis.body);
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный формат экспорта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось выгрузить semantic export проекта.");
}
});
projectsRouter.get("/:projectId/market-enrichment/latest", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
response.json({
marketEnrichment: await getMarketEnrichmentContract(projectId)
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось загрузить market enrichment проекта.");
}
});
projectsRouter.post("/:projectId/market-enrichment", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const marketEnrichment = await runMarketEnrichment(projectId);
if (!marketEnrichment.semanticRunId) {
sendError(response, 404, "Сначала нужен semantic analysis проекта.");
return;
}
response.status(202).json({
marketEnrichment
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось запустить market enrichment проекта.");
}
});
projectsRouter.get("/:projectId/scope-layouts/:layoutKey", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const layoutKey = scopeLayoutKeySchema.parse(request.params.layoutKey);
try {
const rawLayout = await storageProvider.getObjectText(getScopeLayoutObjectKey(projectId, layoutKey));
const layout = scopeLayoutSchema.parse(JSON.parse(rawLayout));
response.json({ layout });
} catch {
response.json({ layout: null });
}
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный ключ layout.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось загрузить layout workflow.");
}
});
projectsRouter.put("/:projectId/scope-layouts/:layoutKey", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const layoutKey = scopeLayoutKeySchema.parse(request.params.layoutKey);
const layout = scopeLayoutSchema.parse(request.body.layout);
await storageProvider.putObject({
body: JSON.stringify(layout),
contentType: "application/json; charset=utf-8",
key: getScopeLayoutObjectKey(projectId, layoutKey)
});
response.json({ layout });
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь layout workflow.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось сохранить layout workflow.");
}
});
projectsRouter.patch("/:projectId/pages/selection", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const input = updatePageSelectionSchema.parse(request.body);
const result = await updateProjectPageSelection(projectId, input.pageIds, input.selected);
response.json(result);
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь выбор страниц.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось обновить выбор страниц.");
}
});
projectsRouter.post("/:projectId/pages/:pageId/workspace", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const pageId = projectIdSchema.parse(request.params.pageId);
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
if (!workspace) {
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
return;
}
response.status(201).json({
workspace
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id страницы.");
return;
}
if (error instanceof PageWorkspaceError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось открыть рабочую зону страницы.");
}
});
projectsRouter.patch("/:projectId/pages/:pageId/workspace", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const pageId = projectIdSchema.parse(request.params.pageId);
const input = updatePageWorkspaceSchema.parse(request.body);
const workspace = await savePageWorkspace(projectId, pageId, input.workingText);
if (!workspace) {
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
return;
}
response.json({
workspace
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь рабочий текст страницы.");
return;
}
if (error instanceof PageWorkspaceError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось сохранить рабочий draft страницы.");
}
});
projectsRouter.post("/:projectId/pages/:pageId/workspace/reset", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const pageId = projectIdSchema.parse(request.params.pageId);
const workspace = await resetPageWorkspace(projectId, pageId);
if (!workspace) {
sendError(response, 404, "Страница не найдена в последнем успешном скане проекта.");
return;
}
response.json({
workspace
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id страницы.");
return;
}
if (error instanceof PageWorkspaceError) {
sendError(response, 400, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось сбросить рабочий draft страницы.");
}
});
projectsRouter.patch("/:projectId", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const input = updateProjectNameSchema.parse(request.body);
const project = await updateProjectName(projectId, input.name);
if (!project) {
sendError(response, 404, "Проект не найден.");
return;
}
response.json({
project
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь название проекта.");
return;
}
if (error instanceof ProjectNameConflictError) {
sendError(response, 409, error.message);
return;
}
console.error(error);
sendError(response, 500, "Не удалось переименовать проект.");
}
});
projectsRouter.delete("/:projectId", async (request, response) => {
try {
const projectId = projectIdSchema.parse(request.params.projectId);
const deleted = await deleteProject(projectId);
if (!deleted) {
sendError(response, 404, "Проект не найден.");
return;
}
response.status(204).send();
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось удалить проект.");
}
});

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,63 @@
import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { pool } from "../db/client.js";
const currentFile = fileURLToPath(import.meta.url);
const migrationsDir = path.resolve(path.dirname(currentFile), "../../migrations");
async function ensureMigrationsTable() {
await pool.query(`
create table if not exists schema_migrations (
id text primary key,
applied_at timestamptz not null default now()
);
`);
}
async function hasMigration(id: string) {
const result = await pool.query<{ exists: boolean }>(
"select exists(select 1 from schema_migrations where id = $1) as exists",
[id]
);
return result.rows[0]?.exists ?? false;
}
async function applyMigration(id: string, sql: string) {
await pool.query("begin");
try {
await pool.query(sql);
await pool.query("insert into schema_migrations (id) values ($1)", [id]);
await pool.query("commit");
} catch (error) {
await pool.query("rollback");
throw error;
}
}
async function main() {
await ensureMigrationsTable();
const files = (await readdir(migrationsDir))
.filter((file) => file.endsWith(".sql"))
.sort((a, b) => a.localeCompare(b));
for (const file of files) {
if (await hasMigration(file)) {
console.log(`Skipping ${file}`);
continue;
}
const sql = await readFile(path.join(migrationsDir, file), "utf8");
await applyMigration(file, sql);
console.log(`Applied ${file}`);
}
await pool.end();
}
main().catch(async (error) => {
console.error(error);
await pool.end();
process.exitCode = 1;
});

View File

@ -0,0 +1,764 @@
import { env } from "../config/env.js";
import { pool } from "../db/client.js";
export type ExternalServiceId =
| "yandex_cloud"
| "yandex_metrica"
| "yandex_search_serp"
| "yandex_webmaster"
| "yandex_wordstat";
type ExternalServiceStatus = "configured" | "disabled" | "missing_required" | "not_implemented";
type ExternalServiceFieldType = "number" | "password" | "text" | "url";
type ExternalServiceField = {
id: string;
label: string;
type: ExternalServiceFieldType;
required: boolean;
placeholder: string;
help: string;
};
type ExternalServiceDefinition = {
id: ExternalServiceId;
label: string;
description: string;
implementationStatus: "active" | "planned";
defaultMode: string;
modes: Array<{
id: string;
label: string;
}>;
docs: Array<{
label: string;
url: string;
}>;
publicFields: ExternalServiceField[];
secretFields: ExternalServiceField[];
};
type ExternalServiceSettingsRow = {
service_id: ExternalServiceId;
enabled: boolean;
mode: string;
public_config: Record<string, unknown>;
secret_config: Record<string, unknown>;
created_at: Date;
updated_at: Date;
};
export type ExternalServiceSettings = {
schemaVersion: "external-service-settings.v1";
services: Array<{
id: ExternalServiceId;
label: string;
description: string;
implementationStatus: ExternalServiceDefinition["implementationStatus"];
status: ExternalServiceStatus;
enabled: boolean;
mode: string;
modes: ExternalServiceDefinition["modes"];
dependsOn: ExternalServiceId[];
docs: ExternalServiceDefinition["docs"];
publicFields: ExternalServiceField[];
secretFields: ExternalServiceField[];
publicConfig: Record<string, string>;
secrets: Record<
string,
{
configured: boolean;
}
>;
missingFields: string[];
updatedAt: string | null;
}>;
};
export type UpdateExternalServiceSettingsInput = {
enabled: boolean;
mode: string;
publicConfig?: Record<string, unknown>;
secrets?: Record<string, unknown>;
};
export type WordstatRuntimeConfig =
| {
provider: "disabled";
mode: "disabled";
}
| {
provider: "mcp_kv";
mode: "mcp_kv";
mcpEndpoint: string;
}
| {
provider: "yandex_api";
mode: "yandex_search_api";
apiBaseUrl: string;
apiToken: string;
authType: "api_key" | "iam_token";
folderId: string;
regionIds: string[];
devices: string[];
numPhrases: number;
};
const WORDSTAT_BASE_URL = "https://searchapi.api.cloud.yandex.net/v2/wordstat";
const WEB_SEARCH_BASE_URL = "https://searchapi.api.cloud.yandex.net/v2/web/search";
const WEBMASTER_BASE_URL = "https://api.webmaster.yandex.net/v4";
const METRICA_BASE_URL = "https://api-metrika.yandex.net";
const serviceDefinitions: Record<ExternalServiceId, ExternalServiceDefinition> = {
yandex_cloud: {
id: "yandex_cloud",
label: "Yandex Cloud credentials",
description: "Один общий credential для Search API слоёв: Wordstat сейчас, SERP позже.",
implementationStatus: "active",
defaultMode: "api_key",
modes: [
{ id: "disabled", label: "Отключено" },
{ id: "api_key", label: "API key" },
{ id: "iam_token", label: "IAM token" }
],
docs: [
{
label: "API keys",
url: "https://yandex.cloud/en/docs/iam/concepts/authorization/api-key"
},
{
label: "Folder ID",
url: "https://yandex.cloud/en/docs/resource-manager/operations/folder/get-id"
}
],
publicFields: [
{
id: "folderId",
label: "Yandex Cloud folderId",
type: "text",
required: true,
placeholder: "b1g...",
help: "ID каталога Yandex Cloud. Его используют Wordstat и SERP Search API."
}
],
secretFields: [
{
id: "apiKey",
label: "API key",
type: "password",
required: false,
placeholder: "Api-Key ...",
help: "Yandex Cloud API key сервисного аккаунта с доступом к Search API."
},
{
id: "iamToken",
label: "IAM token",
type: "password",
required: false,
placeholder: "Bearer token",
help: "Альтернатива API key. Для prototype можно хранить локально, позже уйдёт в vault."
}
]
},
yandex_wordstat: {
id: "yandex_wordstat",
label: "Yandex Wordstat",
description: "Частотность, похожие запросы, динамика и региональная раскладка через Yandex Search API Wordstat.",
implementationStatus: "active",
defaultMode: "yandex_search_api",
modes: [
{ id: "disabled", label: "Отключено" },
{ id: "yandex_search_api", label: "Yandex Search API Wordstat" },
{ id: "mcp_kv", label: "MCP/adapter endpoint" }
],
docs: [
{
label: "Wordstat GetTop",
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/Wordstat/getTop"
},
{
label: "Regions",
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/Wordstat/getRegionsDistribution"
},
{
label: "Dynamics",
url: "https://aistudio.yandex.ru/docs/ru/search-api/api-ref/Wordstat/getDynamics"
}
],
publicFields: [
{
id: "apiBaseUrl",
label: "API base URL",
type: "url",
required: true,
placeholder: WORDSTAT_BASE_URL,
help: "Root endpoint Wordstat v2. Методы вызываются как /topRequests, /regions, /dynamics."
},
{
id: "regionIds",
label: "Region IDs",
type: "text",
required: false,
placeholder: "225",
help: "Через запятую. 225 — Россия; можно заменить на нужные регионы после RegionsTree."
},
{
id: "devices",
label: "Devices",
type: "text",
required: false,
placeholder: "DEVICE_ALL",
help: "Через запятую. Для первого слоя держим DEVICE_ALL."
},
{
id: "numPhrases",
label: "Phrases per seed",
type: "number",
required: false,
placeholder: "50",
help: "Сколько Wordstat rows запрашивать для одной seed-фразы."
},
{
id: "mcpEndpoint",
label: "MCP endpoint",
type: "url",
required: false,
placeholder: "http://localhost:4120/wordstat/collect",
help: "Используется только в режиме MCP/adapter endpoint."
}
],
secretFields: []
},
yandex_search_serp: {
id: "yandex_search_serp",
label: "Yandex SERP",
description: "Поисковая выдача, конкуренты, сниппеты и типы страниц через Yandex Search API WebSearch.",
implementationStatus: "planned",
defaultMode: "yandex_search_api",
modes: [
{ id: "disabled", label: "Отключено" },
{ id: "yandex_search_api", label: "Yandex Search API WebSearch" }
],
docs: [
{
label: "WebSearch.Search",
url: "https://aistudio.yandex.ru/docs/en/search-api/api-ref/WebSearch/search"
}
],
publicFields: [
{
id: "apiBaseUrl",
label: "API endpoint",
type: "url",
required: true,
placeholder: WEB_SEARCH_BASE_URL,
help: "REST endpoint для WebSearch.Search."
},
{
id: "searchType",
label: "Search type",
type: "text",
required: false,
placeholder: "SEARCH_TYPE_RU",
help: "Для российского рынка держим SEARCH_TYPE_RU."
}
],
secretFields: []
},
yandex_webmaster: {
id: "yandex_webmaster",
label: "Yandex Webmaster",
description: "Индексация, ошибки обхода, sitemap, canonical и статусы страниц после Apply.",
implementationStatus: "planned",
defaultMode: "oauth",
modes: [
{ id: "disabled", label: "Отключено" },
{ id: "oauth", label: "OAuth token" }
],
docs: [
{
label: "Webmaster API",
url: "https://yandex.ru/dev/webmaster/doc/en/"
}
],
publicFields: [
{
id: "apiBaseUrl",
label: "API base URL",
type: "url",
required: true,
placeholder: WEBMASTER_BASE_URL,
help: "Базовый URL Webmaster API."
},
{
id: "hostId",
label: "Host ID",
type: "text",
required: false,
placeholder: "https:example.com:443",
help: "ID сайта в Webmaster, можно заполнить позже после discovery."
}
],
secretFields: [
{
id: "oauthToken",
label: "OAuth token",
type: "password",
required: true,
placeholder: "OAuth ...",
help: "Токен доступа Webmaster API."
}
]
},
yandex_metrica: {
id: "yandex_metrica",
label: "Yandex Metrica",
description: "Фактические показы, клики, CTR, цели и обратная связь после SEO-правок.",
implementationStatus: "planned",
defaultMode: "oauth",
modes: [
{ id: "disabled", label: "Отключено" },
{ id: "oauth", label: "OAuth token" }
],
docs: [
{
label: "Metrica API",
url: "https://yandex.ru/dev/metrika/doc/api2/concept/about.html"
}
],
publicFields: [
{
id: "apiBaseUrl",
label: "API base URL",
type: "url",
required: true,
placeholder: METRICA_BASE_URL,
help: "Базовый URL Metrica API."
},
{
id: "counterId",
label: "Counter ID",
type: "text",
required: true,
placeholder: "12345678",
help: "ID счётчика Метрики."
}
],
secretFields: [
{
id: "oauthToken",
label: "OAuth token",
type: "password",
required: true,
placeholder: "OAuth ...",
help: "Токен Metrica API."
}
]
}
};
function toStringRecord(value: Record<string, unknown> | null | undefined) {
const output: Record<string, string> = {};
for (const [key, item] of Object.entries(value ?? {})) {
if (item === null || item === undefined) {
continue;
}
output[key] = String(item);
}
return output;
}
function getFieldDefault(serviceId: ExternalServiceId, fieldId: string) {
if (serviceId === "yandex_cloud") {
return "";
}
if (serviceId === "yandex_wordstat") {
const defaults: Record<string, string> = {
apiBaseUrl: WORDSTAT_BASE_URL,
devices: "DEVICE_ALL",
numPhrases: "50",
regionIds: "225"
};
return defaults[fieldId] ?? "";
}
if (serviceId === "yandex_search_serp") {
const defaults: Record<string, string> = {
apiBaseUrl: WEB_SEARCH_BASE_URL,
searchType: "SEARCH_TYPE_RU"
};
return defaults[fieldId] ?? "";
}
if (serviceId === "yandex_webmaster") {
return fieldId === "apiBaseUrl" ? WEBMASTER_BASE_URL : "";
}
if (serviceId === "yandex_metrica") {
return fieldId === "apiBaseUrl" ? METRICA_BASE_URL : "";
}
return "";
}
function getDependsOn(serviceId: ExternalServiceId, mode: string): ExternalServiceId[] {
if ((serviceId === "yandex_wordstat" || serviceId === "yandex_search_serp") && mode === "yandex_search_api") {
return ["yandex_cloud"];
}
return [];
}
function getPublicConfig(definition: ExternalServiceDefinition, row: ExternalServiceSettingsRow | null) {
const config = toStringRecord(row?.public_config);
for (const field of definition.publicFields) {
if (!config[field.id]) {
config[field.id] = getFieldDefault(definition.id, field.id);
}
}
return config;
}
function getMissingFields(
definition: ExternalServiceDefinition,
enabled: boolean,
mode: string,
publicConfig: Record<string, string>,
secretConfig: Record<string, unknown>,
dependencyStatuses: Map<ExternalServiceId, ExternalServiceStatus> = new Map()
) {
if (!enabled || mode === "disabled") {
return [];
}
const missing: string[] = [];
for (const field of definition.publicFields) {
if (field.id === "mcpEndpoint" && mode !== "mcp_kv") {
continue;
}
if (field.id !== "mcpEndpoint" && mode === "mcp_kv") {
continue;
}
if (field.required && !publicConfig[field.id]?.trim()) {
missing.push(field.id);
}
}
for (const dependencyId of getDependsOn(definition.id, mode)) {
if (dependencyStatuses.get(dependencyId) !== "configured") {
missing.push(dependencyId);
}
}
if (definition.id === "yandex_cloud" && (mode === "api_key" || mode === "iam_token")) {
const hasApiKey = typeof secretConfig.apiKey === "string" && secretConfig.apiKey.trim().length > 0;
const hasIamToken = typeof secretConfig.iamToken === "string" && secretConfig.iamToken.trim().length > 0;
if (mode === "api_key" && !hasApiKey) {
missing.push("apiKey");
}
if (mode === "iam_token" && !hasIamToken) {
missing.push("iamToken");
}
} else {
for (const field of definition.secretFields) {
const value = secretConfig[field.id];
if (field.required && !(typeof value === "string" && value.trim().length > 0)) {
missing.push(field.id);
}
}
}
return missing;
}
function getStatus(
definition: ExternalServiceDefinition,
enabled: boolean,
mode: string,
missingFields: string[]
): ExternalServiceStatus {
if (!enabled || mode === "disabled") {
return "disabled";
}
if (definition.implementationStatus === "planned") {
return "not_implemented";
}
return missingFields.length === 0 ? "configured" : "missing_required";
}
function mapRow(
definition: ExternalServiceDefinition,
row: ExternalServiceSettingsRow | null,
dependencyStatuses: Map<ExternalServiceId, ExternalServiceStatus> = new Map()
) {
const enabled = row?.enabled ?? false;
const mode = row?.mode ?? "disabled";
const publicConfig = getPublicConfig(definition, row);
const secretConfig = row?.secret_config ?? {};
const secrets = Object.fromEntries(
definition.secretFields.map((field) => [
field.id,
{
configured: (() => {
const value = secretConfig[field.id];
return typeof value === "string" && value.trim().length > 0;
})()
}
])
);
const missingFields = getMissingFields(definition, enabled, mode, publicConfig, secretConfig, dependencyStatuses);
return {
id: definition.id,
label: definition.label,
description: definition.description,
implementationStatus: definition.implementationStatus,
status: getStatus(definition, enabled, mode, missingFields),
enabled,
mode,
modes: definition.modes,
dependsOn: getDependsOn(definition.id, mode),
docs: definition.docs,
publicFields: definition.publicFields,
secretFields: definition.secretFields,
publicConfig,
secrets,
missingFields,
updatedAt: row?.updated_at.toISOString() ?? null
};
}
async function getSettingsRows() {
const result = await pool.query<ExternalServiceSettingsRow>(
`
select service_id, enabled, mode, public_config, secret_config, created_at, updated_at
from external_service_settings;
`
);
return new Map(result.rows.map((row) => [row.service_id, row]));
}
function getDependencyStatuses(rows: Map<ExternalServiceId, ExternalServiceSettingsRow>) {
const cloudDefinition = serviceDefinitions.yandex_cloud;
const cloudRow = rows.get("yandex_cloud") ?? null;
const cloudPublicConfig = getPublicConfig(cloudDefinition, cloudRow);
const cloudSecretConfig = cloudRow?.secret_config ?? {};
const cloudEnabled = cloudRow?.enabled ?? false;
const cloudMode = cloudRow?.mode ?? "disabled";
const cloudMissingFields = getMissingFields(
cloudDefinition,
cloudEnabled,
cloudMode,
cloudPublicConfig,
cloudSecretConfig
);
return new Map<ExternalServiceId, ExternalServiceStatus>([
["yandex_cloud", getStatus(cloudDefinition, cloudEnabled, cloudMode, cloudMissingFields)]
]);
}
export async function getExternalServiceSettings(): Promise<ExternalServiceSettings> {
const rows = await getSettingsRows();
const dependencyStatuses = getDependencyStatuses(rows);
return {
schemaVersion: "external-service-settings.v1",
services: (Object.keys(serviceDefinitions) as ExternalServiceId[]).map((serviceId) =>
mapRow(serviceDefinitions[serviceId], rows.get(serviceId) ?? null, dependencyStatuses)
)
};
}
function cleanPublicConfig(definition: ExternalServiceDefinition, value: Record<string, unknown> | undefined) {
const allowedFields = new Set(definition.publicFields.map((field) => field.id));
const output: Record<string, string> = {};
for (const [key, item] of Object.entries(value ?? {})) {
if (!allowedFields.has(key) || item === null || item === undefined) {
continue;
}
output[key] = String(item).trim();
}
return output;
}
function mergeSecrets(
definition: ExternalServiceDefinition,
currentSecrets: Record<string, unknown>,
incomingSecrets: Record<string, unknown> | undefined
) {
const allowedFields = new Set(definition.secretFields.map((field) => field.id));
const output: Record<string, string> = {};
for (const [key, item] of Object.entries(currentSecrets)) {
if (allowedFields.has(key) && typeof item === "string" && item.trim()) {
output[key] = item;
}
}
for (const [key, item] of Object.entries(incomingSecrets ?? {})) {
if (!allowedFields.has(key)) {
continue;
}
const text = String(item ?? "").trim();
if (text) {
output[key] = text;
}
}
return output;
}
export async function updateExternalServiceSettings(
serviceId: ExternalServiceId,
input: UpdateExternalServiceSettingsInput
) {
const definition = serviceDefinitions[serviceId];
const existing = (
await pool.query<ExternalServiceSettingsRow>(
`
select service_id, enabled, mode, public_config, secret_config, created_at, updated_at
from external_service_settings
where service_id = $1;
`,
[serviceId]
)
).rows[0];
const mode = definition.modes.some((item) => item.id === input.mode) ? input.mode : definition.defaultMode;
const publicConfig = cleanPublicConfig(definition, input.publicConfig);
const secretConfig = input.enabled && mode !== "disabled" ? mergeSecrets(definition, existing?.secret_config ?? {}, input.secrets) : {};
const result = await pool.query<ExternalServiceSettingsRow>(
`
insert into external_service_settings (service_id, enabled, mode, public_config, secret_config, updated_at)
values ($1, $2, $3, $4, $5, now())
on conflict (service_id) do update
set enabled = excluded.enabled,
mode = excluded.mode,
public_config = excluded.public_config,
secret_config = excluded.secret_config,
updated_at = now()
returning service_id, enabled, mode, public_config, secret_config, created_at, updated_at;
`,
[serviceId, input.enabled, mode, publicConfig, secretConfig]
);
const row = result.rows[0];
if (!row) {
throw new Error("Не удалось сохранить настройки внешнего сервиса.");
}
const rows = await getSettingsRows();
const dependencyStatuses = getDependencyStatuses(rows);
return mapRow(definition, row, dependencyStatuses);
}
function splitCsv(value: string | undefined, fallback: string[]) {
const values = (value ?? "")
.split(",")
.map((item) => item.trim())
.filter(Boolean);
return values.length > 0 ? values : fallback;
}
function getNumber(value: string | undefined, fallback: number) {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? Math.round(parsed) : fallback;
}
export async function getWordstatRuntimeConfig(): Promise<WordstatRuntimeConfig> {
const rows = await getSettingsRows();
const row = rows.get("yandex_wordstat");
const cloudRow = rows.get("yandex_cloud");
const definition = serviceDefinitions.yandex_wordstat;
const dependencyStatuses = getDependencyStatuses(rows);
const mapped = mapRow(definition, row ?? null, dependencyStatuses);
if (row?.enabled && row.mode === "mcp_kv") {
const endpoint = mapped.publicConfig.mcpEndpoint?.trim();
if (endpoint) {
return {
provider: "mcp_kv",
mode: "mcp_kv",
mcpEndpoint: endpoint
};
}
}
if (row?.enabled && row.mode === "yandex_search_api" && mapped.status === "configured") {
const cloudPublicConfig = getPublicConfig(serviceDefinitions.yandex_cloud, cloudRow ?? null);
const cloudSecretConfig = cloudRow?.secret_config ?? {};
const apiKey = typeof cloudSecretConfig.apiKey === "string" ? cloudSecretConfig.apiKey.trim() : "";
const iamToken = typeof cloudSecretConfig.iamToken === "string" ? cloudSecretConfig.iamToken.trim() : "";
return {
provider: "yandex_api",
mode: "yandex_search_api",
apiBaseUrl: mapped.publicConfig.apiBaseUrl || WORDSTAT_BASE_URL,
apiToken: apiKey || iamToken,
authType: apiKey ? "api_key" : "iam_token",
folderId: cloudPublicConfig.folderId,
regionIds: splitCsv(mapped.publicConfig.regionIds, ["225"]),
devices: splitCsv(mapped.publicConfig.devices, ["DEVICE_ALL"]),
numPhrases: getNumber(mapped.publicConfig.numPhrases, 50)
};
}
if (env.marketProviders.wordstat.provider === "mcp_kv" && env.marketProviders.wordstat.mcpEndpoint) {
return {
provider: "mcp_kv",
mode: "mcp_kv",
mcpEndpoint: env.marketProviders.wordstat.mcpEndpoint
};
}
if (
env.marketProviders.wordstat.provider === "yandex_api" &&
env.marketProviders.wordstat.apiBaseUrl &&
env.marketProviders.wordstat.apiToken &&
env.marketProviders.wordstat.folderId
) {
return {
provider: "yandex_api",
mode: "yandex_search_api",
apiBaseUrl: env.marketProviders.wordstat.apiBaseUrl,
apiToken: env.marketProviders.wordstat.apiToken,
authType: env.marketProviders.wordstat.authType,
folderId: env.marketProviders.wordstat.folderId,
regionIds: env.marketProviders.wordstat.regionIds,
devices: env.marketProviders.wordstat.devices,
numPhrases: env.marketProviders.wordstat.numPhrases
};
}
return {
provider: "disabled",
mode: "disabled"
};
}

View File

@ -0,0 +1,63 @@
import { Router } from "express";
import type { Response } from "express";
import { z } from "zod";
import {
getExternalServiceSettings,
updateExternalServiceSettings,
type ExternalServiceId
} from "./externalServiceSettings.js";
export const settingsRouter = Router();
const externalServiceIdSchema = z.enum([
"yandex_cloud",
"yandex_metrica",
"yandex_search_serp",
"yandex_webmaster",
"yandex_wordstat"
]);
const updateExternalServiceSettingsSchema = z.object({
enabled: z.boolean(),
mode: z.string().trim().min(1),
publicConfig: z.record(z.string(), z.unknown()).optional(),
secrets: z.record(z.string(), z.unknown()).optional()
});
function sendError(response: Response, status: number, message: string) {
response.status(status).json({
error: {
message
}
});
}
settingsRouter.get("/external-services", async (_request, response) => {
try {
response.json({
externalServices: await getExternalServiceSettings()
});
} catch (error) {
console.error(error);
sendError(response, 500, "Не удалось загрузить настройки внешних сервисов.");
}
});
settingsRouter.patch("/external-services/:serviceId", async (request, response) => {
try {
const serviceId = externalServiceIdSchema.parse(request.params.serviceId) as ExternalServiceId;
const input = updateExternalServiceSettingsSchema.parse(request.body);
response.json({
service: await updateExternalServiceSettings(serviceId, input)
});
} catch (error) {
if (error instanceof z.ZodError) {
sendError(response, 400, error.issues[0]?.message ?? "Проверь настройки внешнего сервиса.");
return;
}
console.error(error);
sendError(response, 500, "Не удалось сохранить настройки внешнего сервиса.");
}
});

View File

@ -0,0 +1,171 @@
import {
CopyObjectCommand,
CreateBucketCommand,
DeleteObjectsCommand,
GetObjectCommand,
HeadBucketCommand,
ListObjectsV2Command,
PutObjectCommand,
S3Client
} from "@aws-sdk/client-s3";
import { env } from "../config/env.js";
import type { PutObjectInput, StorageObjectSummary, StorageProvider } from "./StorageProvider.js";
export class S3StorageProvider implements StorageProvider {
private readonly client = new S3Client({
endpoint: env.s3.endpoint,
region: env.s3.region,
forcePathStyle: env.s3.forcePathStyle,
credentials: {
accessKeyId: env.s3.accessKeyId,
secretAccessKey: env.s3.secretAccessKey
}
});
async ensureReady() {
try {
await this.client.send(new HeadBucketCommand({ Bucket: env.s3.bucket }));
} catch {
await this.client.send(new CreateBucketCommand({ Bucket: env.s3.bucket }));
}
}
async putObject(input: PutObjectInput) {
await this.ensureReady();
await this.client.send(
new PutObjectCommand({
Bucket: env.s3.bucket,
Key: input.key,
Body: input.body,
ContentType: input.contentType
})
);
}
async getObject(key: string) {
await this.ensureReady();
const result = await this.client.send(
new GetObjectCommand({
Bucket: env.s3.bucket,
Key: key
})
);
const bytes = result.Body ? await result.Body.transformToByteArray() : new Uint8Array();
return {
body: bytes,
contentType: result.ContentType
};
}
async getObjectText(key: string) {
const result = await this.getObject(key);
return new TextDecoder().decode(result.body);
}
async listPrefix(prefix: string) {
let continuationToken: string | undefined;
const objects: StorageObjectSummary[] = [];
await this.ensureReady();
do {
const listedObjects = await this.client.send(
new ListObjectsV2Command({
Bucket: env.s3.bucket,
Prefix: prefix,
ContinuationToken: continuationToken
})
);
for (const object of listedObjects.Contents ?? []) {
if (!object.Key) {
continue;
}
objects.push({
key: object.Key,
size: object.Size ?? 0,
lastModified: object.LastModified?.toISOString(),
etag: object.ETag
});
}
continuationToken = listedObjects.NextContinuationToken;
} while (continuationToken);
return objects;
}
async copyPrefix(sourcePrefix: string, destinationPrefix: string) {
let continuationToken: string | undefined;
let copiedObjects = 0;
await this.ensureReady();
do {
const listedObjects = await this.client.send(
new ListObjectsV2Command({
Bucket: env.s3.bucket,
Prefix: sourcePrefix,
ContinuationToken: continuationToken
})
);
for (const object of listedObjects.Contents ?? []) {
if (!object.Key) {
continue;
}
const sourceKey = object.Key;
const destinationKey = `${destinationPrefix}${sourceKey.slice(sourcePrefix.length)}`;
await this.client.send(
new CopyObjectCommand({
Bucket: env.s3.bucket,
CopySource: `${env.s3.bucket}/${encodeURIComponent(sourceKey).replace(/%2F/g, "/")}`,
Key: destinationKey
})
);
copiedObjects += 1;
}
continuationToken = listedObjects.NextContinuationToken;
} while (continuationToken);
return copiedObjects;
}
async deletePrefix(prefix: string) {
let continuationToken: string | undefined;
await this.ensureReady();
do {
const listedObjects = await this.client.send(
new ListObjectsV2Command({
Bucket: env.s3.bucket,
Prefix: prefix,
ContinuationToken: continuationToken
})
);
const objects = listedObjects.Contents?.map((object) => ({ Key: object.Key })).filter((object) => object.Key);
if (objects && objects.length > 0) {
await this.client.send(
new DeleteObjectsCommand({
Bucket: env.s3.bucket,
Delete: {
Objects: objects
}
})
);
}
continuationToken = listedObjects.NextContinuationToken;
} while (continuationToken);
}
}

View File

@ -0,0 +1,27 @@
export type PutObjectInput = {
key: string;
body: string | Uint8Array;
contentType?: string;
};
export type StorageObjectSummary = {
key: string;
size: number;
lastModified?: string;
etag?: string;
};
export type StorageObject = {
body: Uint8Array;
contentType?: string;
};
export interface StorageProvider {
ensureReady(): Promise<void>;
putObject(input: PutObjectInput): Promise<void>;
getObject(key: string): Promise<StorageObject>;
getObjectText(key: string): Promise<string>;
listPrefix(prefix: string): Promise<StorageObjectSummary[]>;
copyPrefix(sourcePrefix: string, destinationPrefix: string): Promise<number>;
deletePrefix(prefix: string): Promise<void>;
}

View File

@ -0,0 +1,3 @@
import { S3StorageProvider } from "./S3StorageProvider.js";
export const storageProvider = new S3StorageProvider();

View File

@ -0,0 +1,823 @@
import fs from "node:fs/promises";
import path from "node:path";
import { pool } from "../db/client.js";
import { storageProvider } from "../storage/index.js";
import { annotatePreviewTargets, getPreviewTargetSelector, getPreviewTargetSeeds } from "./previewTargets.js";
type PageWorkspaceSourceRow = {
project_id: string;
page_id: string;
scan_version_id: string;
source_path: string;
url_path: string | null;
title: string | null;
description: string | null;
raw: Record<string, unknown>;
source_type: string;
source_config: Record<string, unknown>;
};
type DraftRow = {
id: string;
project_id: string;
page_id: string;
status: string;
created_at: Date;
updated_at: Date;
};
type DraftItemRow = {
id: string;
draft_id: string;
field: string;
original_snapshot_key: string | null;
working_snapshot_key: string | null;
working_text: string | null;
created_at: Date;
updated_at: Date;
};
type WorkspaceSnapshot = {
schemaVersion: "page-workspace.v2";
projectId: string;
pageId: string;
scanVersionId: string;
sourcePath: string;
urlPath: string | null;
title: string | null;
description: string | null;
headings: Array<{ level: number; text: string }>;
originalText: string;
originalHtml: string;
previewHtml: string;
sections: PageWorkspaceSection[];
createdAt: string;
};
export type PageWorkspaceSection = {
id: string;
kind: "meta" | "content_block" | "heading_group" | "media";
title: string;
selector: string | null;
originalText: string;
workingText: string;
charCount: number;
summary: string;
sourceHint: string;
};
export type PageWorkspace = {
projectId: string;
pageId: string;
page: {
scanVersionId: string;
sourcePath: string;
urlPath: string | null;
title: string | null;
description: string | null;
headings: Array<{ level: number; text: string }>;
};
draft: {
id: string;
status: string;
createdAt: string;
updatedAt: string;
};
item: {
id: string;
field: string;
originalSnapshotKey: string | null;
workingSnapshotKey: string | null;
originalText: string;
workingText: string;
originalTextLength: number;
workingTextLength: number;
updatedAt: string;
};
previewHtml: string;
sections: PageWorkspaceSection[];
};
export class PageWorkspaceError extends Error {
constructor(message: string) {
super(message);
this.name = "PageWorkspaceError";
}
}
const WORKSPACE_FIELD = "page_text";
const MAX_BLOCK_TEXT_LENGTH = 6_000;
const MAX_SECTIONS = 80;
const MAX_MEDIA_SECTIONS = 80;
function toIsoDate(value: Date) {
return value.toISOString();
}
function normalizePath(value: string) {
return value.replace(/\\/g, "/").replace(/^\/+/, "");
}
function getStringConfig(config: Record<string, unknown>, key: string) {
const value = config[key];
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
function decodeHtmlEntities(value: string) {
return value
.replace(/&nbsp;/gi, " ")
.replace(/&amp;/gi, "&")
.replace(/&quot;/gi, "\"")
.replace(/&#39;/gi, "'")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">");
}
function normalizeWhitespace(value: string) {
return decodeHtmlEntities(value).replace(/\s+/g, " ").trim();
}
function stripIgnoredHtmlBlocks(html: string) {
return html
.replace(/<script\b[\s\S]*?<\/script>/gi, " ")
.replace(/<style\b[\s\S]*?<\/style>/gi, " ")
.replace(/<svg\b[\s\S]*?<\/svg>/gi, " ");
}
function getBodyHtml(html: string) {
const bodyMatch = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(html);
return bodyMatch?.[1] ?? html;
}
function getAttribute(attributes: string, name: string) {
const pattern = new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i");
return pattern.exec(attributes)?.[1] ?? null;
}
function getDisplayFilename(source: string | null, fallback: string) {
if (!source) {
return fallback;
}
const cleanSource = source.split(/[?#]/)[0] ?? source;
return path.basename(cleanSource) || source;
}
function htmlToReadableText(html: string) {
const withBreaks = stripIgnoredHtmlBlocks(html)
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<\/(p|div|section|article|header|footer|main|aside|nav|li|ul|ol|h[1-6])>/gi, "\n")
.replace(/<(h[1-6])\b[^>]*>/gi, "\n");
return decodeHtmlEntities(withBreaks.replace(/<[^>]+>/g, " "))
.split(/\n+/)
.map((line) => normalizeWhitespace(line))
.filter(Boolean)
.join("\n")
.slice(0, MAX_BLOCK_TEXT_LENGTH);
}
function normalizeHeadings(raw: Record<string, unknown>) {
const headings = raw.headings;
if (!Array.isArray(headings)) {
return [];
}
return headings
.map((heading) => {
if (typeof heading !== "object" || heading === null) {
return null;
}
const level = "level" in heading && typeof heading.level === "number" ? heading.level : null;
const text = "text" in heading && typeof heading.text === "string" ? heading.text : null;
return level && text ? { level, text } : null;
})
.filter((heading): heading is { level: number; text: string } => Boolean(heading));
}
function getFirstH1Text(raw: Record<string, unknown>) {
return normalizeHeadings(raw).find((heading) => heading.level === 1)?.text ?? "";
}
function countImageTags(html: string) {
return Array.from(getBodyHtml(html).matchAll(/<img\b[^>]*>/gi)).length;
}
function getSectionSummary(text: string) {
const firstMeaningfulLine = text
.split("\n")
.map((line) => line.trim())
.find((line) => line.length > 0);
if (!firstMeaningfulLine) {
return "Пустой текстовый блок.";
}
return firstMeaningfulLine.length > 180 ? `${firstMeaningfulLine.slice(0, 177)}...` : firstMeaningfulLine;
}
function makeSection(input: Omit<PageWorkspaceSection, "charCount" | "summary" | "workingText">): PageWorkspaceSection {
return {
...input,
workingText: input.originalText,
charCount: input.originalText.length,
summary: getSectionSummary(input.originalText)
};
}
function getFirstHeadingText(html: string) {
const headingMatch = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/i.exec(html);
return headingMatch ? htmlToReadableText(headingMatch[2] ?? "") : null;
}
function getSemanticBlockTitle(tagName: string, attributes: string, text: string, fallbackIndex: number) {
const heading = getFirstHeadingText(text);
const id = getAttribute(attributes, "id");
const className = getAttribute(attributes, "class");
const label = getAttribute(attributes, "aria-label");
if (heading) return heading;
if (label) return label;
if (id) return `${tagName}#${id}`;
if (className) return `${tagName}.${className.split(/\s+/).filter(Boolean).slice(0, 2).join(".")}`;
return `Блок ${fallbackIndex}`;
}
function dedupeSections(sections: PageWorkspaceSection[]) {
const seenTexts = new Set<string>();
const dedupedSections: PageWorkspaceSection[] = [];
for (const section of sections) {
const normalizedText = normalizeWhitespace(section.originalText).toLocaleLowerCase();
if (!normalizedText || seenTexts.has(normalizedText)) {
continue;
}
seenTexts.add(normalizedText);
dedupedSections.push(section);
}
return dedupedSections.slice(0, MAX_SECTIONS);
}
function extractSemanticSections(html: string) {
const bodyHtml = getBodyHtml(html);
const blockPattern = /<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi;
const sections: PageWorkspaceSection[] = [];
let match: RegExpExecArray | null;
let index = 1;
while ((match = blockPattern.exec(bodyHtml)) && sections.length < MAX_SECTIONS) {
const tagName = match[1] ?? "section";
const attributes = match[2] ?? "";
const blockHtml = match[3] ?? "";
const text = htmlToReadableText(blockHtml);
if (text.length < 20) {
continue;
}
sections.push(
makeSection({
id: `block-${index}`,
kind: "content_block",
title: getSemanticBlockTitle(tagName, attributes, blockHtml, index),
selector: getPreviewTargetSelector(`block-${index}`),
originalText: text,
sourceHint: `<${tagName}>`
})
);
index += 1;
}
return dedupeSections(sections);
}
function extractHeadingSections(html: string) {
const bodyHtml = getBodyHtml(html);
const headingPattern = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
const headingMatches = Array.from(bodyHtml.matchAll(headingPattern));
const sections: PageWorkspaceSection[] = [];
headingMatches.forEach((headingMatch, index) => {
if (sections.length >= MAX_SECTIONS) {
return;
}
const level = headingMatch[1] ?? "";
const headingHtml = headingMatch[2] ?? "";
const startIndex = headingMatch.index ?? 0;
const nextIndex = headingMatches[index + 1]?.index ?? bodyHtml.length;
const chunkHtml = bodyHtml.slice(startIndex, nextIndex);
const text = htmlToReadableText(chunkHtml);
const headingText = htmlToReadableText(headingHtml) || `H${level}`;
if (text.length < 8) {
return;
}
sections.push(
makeSection({
id: `heading-group-${index + 1}`,
kind: "heading_group",
title: `H${level}: ${headingText}`,
selector: getPreviewTargetSelector(`heading-group-${index + 1}`),
originalText: text,
sourceHint: "heading-группа"
})
);
});
return dedupeSections(sections);
}
function extractImageAltSections(html: string) {
const bodyHtml = getBodyHtml(html);
const imagePattern = /<img\b([^>]*)>/gi;
const sections: PageWorkspaceSection[] = [];
let match: RegExpExecArray | null;
let imageIndex = 0;
while ((match = imagePattern.exec(bodyHtml)) && sections.length < MAX_MEDIA_SECTIONS) {
imageIndex += 1;
const attributes = match[1] ?? "";
const source = getAttribute(attributes, "src");
const alt = getAttribute(attributes, "alt");
const normalizedAlt = normalizeWhitespace(alt ?? "");
const filename = getDisplayFilename(source, `image-${imageIndex}`);
sections.push(
makeSection({
id: `media-alt-${imageIndex}`,
kind: "media",
title: `ALT: ${filename}`,
selector: getPreviewTargetSelector(`media-alt-${imageIndex}`),
originalText: normalizedAlt,
sourceHint: source ?? `img:nth-of-type(${imageIndex})`
})
);
}
return sections;
}
function buildPreviewHtml(html: string) {
const previewStyles = `
<style>
html { background: #fff !important; }
body { min-height: 100vh; }
a { cursor: default !important; }
</style>
`;
let imageIndex = 0;
const targetSeeds = getPreviewTargetSeeds(html);
const withoutScripts = annotatePreviewTargets(html, targetSeeds)
.replace(/<script\b[\s\S]*?<\/script>/gi, "")
.replace(/\son[a-z]+\s*=\s*["'][^"']*["']/gi, "")
.replace(/<h([1-3])\b([^>]*)>/gi, (_match, level: string, attributes: string) => {
return `<h${level}${attributes} data-seo-label="H${level}">`;
})
.replace(/<img\b([^>]*)>/gi, (match, attributes: string) => {
imageIndex += 1;
if (/data-seo-media-index\s*=/i.test(attributes)) {
return match;
}
const isSelfClosing = /\/\s*$/.test(attributes);
const cleanAttributes = attributes.replace(/\/\s*$/, "").trimEnd();
return `<img${cleanAttributes} data-seo-media-index="${imageIndex}"${isSelfClosing ? " /" : ""}>`;
});
if (/<head\b[^>]*>/i.test(withoutScripts)) {
return withoutScripts.replace(/<head\b[^>]*>/i, (headTag) => `${headTag}<base target="_blank">${previewStyles}`);
}
return `<!doctype html><html><head><base target="_blank">${previewStyles}</head><body>${withoutScripts}</body></html>`;
}
function buildSections(page: PageWorkspaceSourceRow, html: string): PageWorkspaceSection[] {
const sections: PageWorkspaceSection[] = [];
if (page.title) {
sections.push(makeSection({
id: "seo-title",
kind: "meta",
title: "Title",
selector: "head > title",
originalText: page.title,
sourceHint: "SEO-поле"
}));
}
if (page.description) {
sections.push(makeSection({
id: "seo-description",
kind: "meta",
title: "Meta description",
selector: "meta[name='description']",
originalText: page.description,
sourceHint: "SEO-поле"
}));
}
const semanticSections = extractSemanticSections(html);
const contentSections = semanticSections.length >= 2 ? semanticSections : extractHeadingSections(html);
const mediaSections = extractImageAltSections(html);
return [...sections, ...contentSections, ...mediaSections];
}
function buildIssueAlignedSections(page: PageWorkspaceSourceRow, html: string): PageWorkspaceSection[] {
const semanticSections = extractSemanticSections(html);
const contentSections = semanticSections.length >= 2 ? semanticSections : extractHeadingSections(html);
const mediaSections = extractImageAltSections(html);
const pageAnchorSelector = contentSections[0]?.selector ?? mediaSections[0]?.selector ?? getPreviewTargetSelector("block-1");
const h1Section = contentSections.find((section) => section.title.toLocaleLowerCase().startsWith("h1:"));
const seoSections: PageWorkspaceSection[] = [
makeSection({
id: "seo-title",
kind: "meta",
title: "Title",
selector: pageAnchorSelector,
originalText: page.title ?? "",
sourceHint: "<title>"
}),
makeSection({
id: "seo-description",
kind: "meta",
title: "Meta description",
selector: pageAnchorSelector,
originalText: page.description ?? "",
sourceHint: "meta[name='description']"
}),
makeSection({
id: "seo-h1",
kind: "heading_group",
title: "H1",
selector: h1Section?.selector ?? pageAnchorSelector,
originalText: getFirstH1Text(page.raw),
sourceHint: "H1"
})
];
return [...seoSections, ...contentSections, ...mediaSections];
}
function buildOriginalText(page: PageWorkspaceSourceRow, sections: PageWorkspaceSection[]) {
const header = [
`Источник: ${page.source_path}`,
page.url_path ? `URL: ${page.url_path}` : null,
`Scan version: ${page.scan_version_id}`
].filter(Boolean);
const sectionText = sections.map((section) => {
return `## ${section.title}\n${section.originalText}`;
});
return [...header, "", ...sectionText].join("\n\n").trim();
}
function extractWorkingSectionTexts(workingText: string, sectionCount: number) {
const headings = Array.from(workingText.matchAll(/^## .+$/gm));
if (headings.length === 0) {
return [];
}
return headings.slice(0, sectionCount).map((heading, index) => {
const start = (heading.index ?? 0) + heading[0].length;
const end = headings[index + 1]?.index ?? workingText.length;
return workingText.slice(start, end).trim();
});
}
async function getLatestPageWorkspaceSource(projectId: string, pageId: string) {
const result = await pool.query<PageWorkspaceSourceRow>(
`
select
sv.project_id,
pv.page_id,
pv.scan_version_id,
pv.source_path,
pv.url_path,
pv.title,
pv.description,
pv.raw,
ps.type as source_type,
ps.config as source_config
from page_versions pv
join scan_versions sv on sv.id = pv.scan_version_id
join project_sources ps on ps.id = sv.source_id
where sv.project_id = $1
and pv.page_id = $2
and sv.status = 'done'
order by sv.completed_at desc nulls last, sv.created_at desc
limit 1;
`,
[projectId, pageId]
);
return result.rows[0] ?? null;
}
async function readPageHtml(page: PageWorkspaceSourceRow) {
if (page.source_type === "browser_folder") {
const importPrefix = getStringConfig(page.source_config, "importPrefix");
if (!importPrefix) {
throw new PageWorkspaceError("У проекта нет importPrefix для чтения HTML из MinIO.");
}
const normalizedPrefix = importPrefix.endsWith("/") ? importPrefix : `${importPrefix}/`;
return storageProvider.getObjectText(`${normalizedPrefix}${normalizePath(page.source_path)}`);
}
if (page.source_type === "local_folder") {
const sourcePath = getStringConfig(page.source_config, "path");
if (!sourcePath) {
throw new PageWorkspaceError("У проекта нет локального пути для чтения HTML.");
}
return fs.readFile(path.join(sourcePath, normalizePath(page.source_path)), "utf8");
}
throw new PageWorkspaceError("Рабочая зона для этого типа источника пока не поддерживается.");
}
async function getActiveDraft(projectId: string, pageId: string) {
const result = await pool.query<DraftRow>(
`
select *
from drafts
where project_id = $1 and page_id = $2 and status = 'active'
order by updated_at desc
limit 1;
`,
[projectId, pageId]
);
return result.rows[0] ?? null;
}
async function getDraftItem(draftId: string) {
const result = await pool.query<DraftItemRow>(
`
select *
from draft_items
where draft_id = $1 and field = $2
order by updated_at desc
limit 1;
`,
[draftId, WORKSPACE_FIELD]
);
return result.rows[0] ?? null;
}
async function readOriginalSnapshot(key: string | null) {
if (!key) {
throw new PageWorkspaceError("У draft нет original snapshot.");
}
const text = await storageProvider.getObjectText(key);
return JSON.parse(text) as WorkspaceSnapshot;
}
function buildWorkspaceResponse(page: PageWorkspaceSourceRow, draft: DraftRow, item: DraftItemRow, snapshot: WorkspaceSnapshot): PageWorkspace {
const workingText = item.working_text ?? snapshot.originalText;
const workingSectionTexts = extractWorkingSectionTexts(workingText, snapshot.sections.length);
const sections = snapshot.sections.map((section, index) => {
const sectionWorkingText = workingSectionTexts[index] ?? section.originalText;
return {
...section,
workingText: sectionWorkingText,
charCount: sectionWorkingText.length,
summary: getSectionSummary(sectionWorkingText)
};
});
return {
projectId: page.project_id,
pageId: page.page_id,
page: {
scanVersionId: page.scan_version_id,
sourcePath: page.source_path,
urlPath: page.url_path,
title: page.title,
description: page.description,
headings: normalizeHeadings(page.raw)
},
draft: {
id: draft.id,
status: draft.status,
createdAt: toIsoDate(draft.created_at),
updatedAt: toIsoDate(draft.updated_at)
},
item: {
id: item.id,
field: item.field,
originalSnapshotKey: item.original_snapshot_key,
workingSnapshotKey: item.working_snapshot_key,
originalText: snapshot.originalText,
workingText,
originalTextLength: snapshot.originalText.length,
workingTextLength: workingText.length,
updatedAt: toIsoDate(item.updated_at)
},
previewHtml: snapshot.previewHtml,
sections
};
}
function buildSnapshot(page: PageWorkspaceSourceRow, html: string): WorkspaceSnapshot {
const sections = buildIssueAlignedSections(page, html);
const originalText = buildOriginalText(page, sections);
return {
schemaVersion: "page-workspace.v2",
projectId: page.project_id,
pageId: page.page_id,
scanVersionId: page.scan_version_id,
sourcePath: page.source_path,
urlPath: page.url_path,
title: page.title,
description: page.description,
headings: normalizeHeadings(page.raw),
originalText,
originalHtml: html,
previewHtml: buildPreviewHtml(html),
sections,
createdAt: new Date().toISOString()
};
}
async function getUpgradedSnapshot(page: PageWorkspaceSourceRow, item: DraftItemRow) {
const snapshot = await readOriginalSnapshot(item.original_snapshot_key);
const imageCount = Math.min(countImageTags(snapshot.originalHtml ?? ""), MAX_MEDIA_SECTIONS);
const mediaSectionCount = snapshot.sections.filter((section) => section.kind === "media").length;
const isCurrentSnapshot =
snapshot.schemaVersion === "page-workspace.v2" &&
"previewHtml" in snapshot &&
snapshot.sections.some((section) => section.id === "seo-title") &&
snapshot.sections.some((section) => section.id === "seo-description") &&
snapshot.sections.some((section) => section.id === "seo-h1") &&
snapshot.sections.some((section) => section.kind === "content_block" || section.kind === "heading_group") &&
(!/<img\b/i.test(snapshot.originalHtml ?? "") || mediaSectionCount >= imageCount) &&
snapshot.sections.every((section) => section.kind === "meta" || section.selector?.includes("data-seo-target-id"));
if (isCurrentSnapshot) {
return snapshot;
}
const html = snapshot.originalHtml || (await readPageHtml(page));
const upgradedSnapshot = buildSnapshot(page, html);
if (item.original_snapshot_key) {
await storageProvider.putObject({
key: item.original_snapshot_key,
body: JSON.stringify(upgradedSnapshot, null, 2),
contentType: "application/json"
});
}
if (!item.working_text || item.working_text === snapshot.originalText) {
await pool.query(
`
update draft_items
set working_text = $1, updated_at = now()
where id = $2;
`,
[upgradedSnapshot.originalText, item.id]
);
item.working_text = upgradedSnapshot.originalText;
item.updated_at = new Date();
}
return upgradedSnapshot;
}
export async function getOrCreatePageWorkspace(projectId: string, pageId: string) {
const page = await getLatestPageWorkspaceSource(projectId, pageId);
if (!page) {
return null;
}
const existingDraft = await getActiveDraft(projectId, pageId);
const existingItem = existingDraft ? await getDraftItem(existingDraft.id) : null;
if (existingDraft && existingItem) {
const snapshot = await getUpgradedSnapshot(page, existingItem);
return buildWorkspaceResponse(page, existingDraft, existingItem, snapshot);
}
const html = await readPageHtml(page);
const snapshot = buildSnapshot(page, html);
const client = await pool.connect();
try {
await client.query("begin");
const draftResult = await client.query<DraftRow>(
`
insert into drafts (project_id, page_id, status)
values ($1, $2, 'active')
returning *;
`,
[projectId, pageId]
);
const draft = draftResult.rows[0];
if (!draft) {
throw new Error("Не удалось создать draft.");
}
const originalSnapshotKey = `projects/${projectId}/drafts/${draft.id}/original-page-workspace.json`;
await storageProvider.putObject({
key: originalSnapshotKey,
body: JSON.stringify(snapshot, null, 2),
contentType: "application/json"
});
const itemResult = await client.query<DraftItemRow>(
`
insert into draft_items (draft_id, field, original_snapshot_key, working_text)
values ($1, $2, $3, $4)
returning *;
`,
[draft.id, WORKSPACE_FIELD, originalSnapshotKey, snapshot.originalText]
);
const item = itemResult.rows[0];
if (!item) {
throw new Error("Не удалось создать draft item.");
}
await client.query("commit");
return buildWorkspaceResponse(page, draft, item, snapshot);
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
}
async function updateWorkspaceText(projectId: string, pageId: string, workingText: string) {
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
if (!workspace) {
return null;
}
const workingSnapshotKey = `projects/${projectId}/drafts/${workspace.draft.id}/working-${Date.now()}.txt`;
await storageProvider.putObject({
key: workingSnapshotKey,
body: workingText,
contentType: "text/plain; charset=utf-8"
});
await pool.query(
`
update draft_items
set working_text = $1, working_snapshot_key = $2, updated_at = now()
where id = $3;
`,
[workingText, workingSnapshotKey, workspace.item.id]
);
await pool.query(
`
update drafts
set updated_at = now()
where id = $1;
`,
[workspace.draft.id]
);
return getOrCreatePageWorkspace(projectId, pageId);
}
export async function savePageWorkspace(projectId: string, pageId: string, workingText: string) {
return updateWorkspaceText(projectId, pageId, workingText);
}
export async function resetPageWorkspace(projectId: string, pageId: string) {
const workspace = await getOrCreatePageWorkspace(projectId, pageId);
if (!workspace) {
return null;
}
return updateWorkspaceText(projectId, pageId, workspace.item.originalText);
}

View File

@ -0,0 +1,162 @@
export type PreviewTargetSeed = {
id: string;
kind: "content_block" | "heading_group" | "media";
selector: string;
sourceHint: string;
targetIndex: number;
};
function normalizeWhitespace(value: string) {
return value.replace(/\s+/g, " ").trim();
}
function getBodyHtml(html: string) {
const bodyMatch = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(html);
return bodyMatch?.[1] ?? html;
}
function getAttribute(attributes: string, name: string) {
const pattern = new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i");
return pattern.exec(attributes)?.[1] ?? null;
}
function htmlToPlainText(html: string) {
return normalizeWhitespace(html.replace(/<script\b[\s\S]*?<\/script>/gi, " ").replace(/<style\b[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " "));
}
function escapeAttribute(value: string) {
return value.replace(/&/g, "&amp;").replace(/"/g, "&quot;");
}
function injectAttribute(openingTag: string, name: string, value: string) {
if (new RegExp(`\\s${name}\\s*=`, "i").test(openingTag)) {
return openingTag;
}
return openingTag.replace(/\/?\s*>$/, (ending) => ` ${name}="${escapeAttribute(value)}"${ending}`);
}
export function getPreviewTargetSelector(targetId: string) {
return `[data-seo-target-id="${targetId}"]`;
}
export function getPreviewTargetSeeds(html: string) {
const bodyHtml = getBodyHtml(html);
const seeds: PreviewTargetSeed[] = [];
const blockPattern = /<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi;
let blockMatch: RegExpExecArray | null;
let blockIndex = 1;
while ((blockMatch = blockPattern.exec(bodyHtml)) && seeds.length < 80) {
const tagName = blockMatch[1] ?? "section";
const blockHtml = blockMatch[3] ?? "";
const text = htmlToPlainText(blockHtml);
if (text.length < 20) {
continue;
}
const id = `block-${blockIndex}`;
seeds.push({
id,
kind: "content_block",
selector: getPreviewTargetSelector(id),
sourceHint: `<${tagName}>`,
targetIndex: blockIndex
});
blockIndex += 1;
}
const headingPattern = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
let headingMatch: RegExpExecArray | null;
let headingIndex = 1;
while ((headingMatch = headingPattern.exec(bodyHtml)) && seeds.length < 120) {
const text = htmlToPlainText(headingMatch[2] ?? "");
if (text.length < 2) {
headingIndex += 1;
continue;
}
const id = `heading-group-${headingIndex}`;
seeds.push({
id,
kind: "heading_group",
selector: getPreviewTargetSelector(id),
sourceHint: `H${headingMatch[1] ?? ""}`,
targetIndex: headingIndex
});
headingIndex += 1;
}
const imagePattern = /<img\b([^>]*)>/gi;
let imageMatch: RegExpExecArray | null;
let imageIndex = 0;
while ((imageMatch = imagePattern.exec(bodyHtml)) && seeds.length < 180) {
imageIndex += 1;
const attributes = imageMatch[1] ?? "";
const id = `media-alt-${imageIndex}`;
seeds.push({
id,
kind: "media",
selector: getPreviewTargetSelector(id),
sourceHint: getAttribute(attributes, "src") ?? `img:nth-of-type(${imageIndex})`,
targetIndex: imageIndex
});
}
return seeds;
}
export function annotatePreviewTargets(html: string, seeds: PreviewTargetSeed[]) {
const blockSeeds = seeds.filter((seed) => seed.kind === "content_block");
const headingSeeds = seeds.filter((seed) => seed.kind === "heading_group");
const mediaSeeds = seeds.filter((seed) => seed.kind === "media");
let blockIndex = 0;
let headingIndex = 0;
let imageIndex = 0;
return html
.replace(/<(section|article|header|main|footer|aside|nav)\b([^>]*)>([\s\S]*?)<\/\1>/gi, (blockHtml, tagName: string, attributes: string, content: string) => {
const text = htmlToPlainText(content);
if (text.length < 20) {
return blockHtml;
}
const seed = blockSeeds[blockIndex];
blockIndex += 1;
if (!seed) {
return blockHtml;
}
const openingTag = `<${tagName}${attributes}>`;
return blockHtml.replace(openingTag, injectAttribute(openingTag, "data-seo-target-id", seed.id));
})
.replace(/<h([1-6])\b([^>]*)>([\s\S]*?)<\/h\1>/gi, (headingHtml, level: string, attributes: string, content: string) => {
const text = htmlToPlainText(content);
if (text.length < 2) {
return headingHtml;
}
const seed = headingSeeds[headingIndex];
headingIndex += 1;
if (!seed) {
return headingHtml;
}
const openingTag = `<h${level}${attributes}>`;
return headingHtml.replace(openingTag, injectAttribute(openingTag, "data-seo-target-id", seed.id));
})
.replace(/<img\b([^>]*)>/gi, (openingTag) => {
imageIndex += 1;
const seed = mediaSeeds.find((candidate) => candidate.targetIndex === imageIndex);
return seed ? injectAttribute(openingTag, "data-seo-target-id", seed.id) : openingTag;
});
}

View File

@ -0,0 +1,12 @@
{
"extends": "../tsconfig.base.json",
"compilerOptions": {
"outDir": "dist",
"rootDir": "src",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"types": ["node"],
"noEmitOnError": true
},
"include": ["src/**/*.ts"]
}

View File

@ -0,0 +1,13 @@
{
"compilerOptions": {
"target": "ES2022",
"useDefineForClassFields": true,
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"isolatedModules": true
}
}