diff --git a/seo_mode/seo_mode/app/src/App.tsx b/seo_mode/seo_mode/app/src/App.tsx index 804af4b..cb6231c 100644 --- a/seo_mode/seo_mode/app/src/App.tsx +++ b/seo_mode/seo_mode/app/src/App.tsx @@ -305,6 +305,23 @@ function formatCount(value: number | null) { return value === null ? "n/a" : numberFormatter.format(value); } +function formatShortDuration(ms: number) { + const totalMinutes = Math.max(0, Math.ceil(ms / 60_000)); + + if (totalMinutes <= 1) { + return "≤1м"; + } + + const hours = Math.floor(totalMinutes / 60); + const minutes = totalMinutes % 60; + + if (hours <= 0) { + return `${minutes}м`; + } + + return minutes > 0 ? `${hours}ч ${minutes}м` : `${hours}ч`; +} + function getRussianPlural(value: number, one: string, few: string, many: string) { const normalized = Math.abs(value); const lastTwo = normalized % 100; @@ -4463,18 +4480,24 @@ function KeywordCurationColumn({ } function KeywordCurationBoard({ + isBusy, + isSaving, isSubmitting, keywordMap, onReconfigure, onRoleChange, + onSave, onSubmit, overrides, stageProjectId }: { + isBusy: boolean; + isSaving: boolean; isSubmitting: boolean; keywordMap: KeywordMapContract; onReconfigure: () => void; onRoleChange: (itemId: string, role: KeywordCurationRole) => void; + onSave: () => void; onSubmit: () => void; overrides: Record; stageProjectId?: string; @@ -4491,7 +4514,8 @@ function KeywordCurationBoard({ const activeItem = activeItemId ? keywordMap.items.find((item) => item.id === activeItemId) ?? null : null; const activeRole = activeItem ? getKeywordCurationRoleForItem(activeItem, overrides) : "validate"; const persistableCount = getKeywordCurationPersistableCount(keywordMap, overrides); - const submitDisabled = isSubmitting || keywordMap.state !== "draft" || persistableCount === 0; + const saveDisabled = isBusy || keywordMap.state !== "draft" || persistableCount === 0; + const submitDisabled = isBusy || keywordMap.state !== "draft" || persistableCount === 0; function getDropRoleAtPoint(x: number, y: number) { const element = document.elementFromPoint(x, y)?.closest("[data-keyword-drop-role]"); @@ -4574,12 +4598,31 @@ function KeywordCurationBoard({
Этап 3 · Распределение ключей Модель распределяет, пользователь правит. Неразобранные не участвуют. + Сформировано: {formatDate(keywordMap.generatedAt)}
- +
+ + +
{keywordCurationColumns.map((column) => ( @@ -5003,15 +5046,19 @@ function getPipelineStageAccess( if (stageIndex === 5) { return { - reason: marketReady && keywordReady ? "Есть спрос и карта фраз, можно выбирать стратегию посадочных." : "Сначала нужен рыночный спрос и карта фраз.", - status: marketReady && keywordReady ? "ready" : "locked", - unlocked: marketReady && keywordReady + reason: strategyReady + ? "Стратегия посадочных сформирована, можно открыть карту посадочных." + : marketReady && keywordReady + ? "Нажми «Формировать стратегию» на этапе ключей и спроса." + : "Сначала нужен рыночный спрос и карта фраз.", + status: strategyReady ? "ready" : "locked", + unlocked: strategyReady }; } if (stageIndex === 6) { return { - reason: strategyReady ? "Стратегия посадочных собрана, можно смотреть план правок." : "Сначала нужна стратегическая база и доказательства.", + reason: strategyReady ? "Стратегия посадочных собрана, можно смотреть план правок." : "План правок откроется после формирования стратегии.", status: strategyReady ? "ready" : "locked", unlocked: strategyReady }; @@ -5230,6 +5277,65 @@ function getMarketStateLabel(state: MarketEnrichmentContract["state"]) { return labels[state]; } +function WordstatQuotaIndicator({ quota }: { quota: NonNullable }) { + const [nowMs, setNowMs] = useState(() => Date.now()); + const limit = Math.max(1, quota.limitPerHour); + const activeReleaseItems = (quota.releaseSchedule ?? []).filter((item) => { + const releasesAtMs = new Date(item.releasesAt).getTime(); + + return Number.isFinite(releasesAtMs) && releasesAtMs > nowMs; + }); + const scheduledUsed = activeReleaseItems.reduce((sum, item) => sum + Math.max(0, item.requestCount), 0); + const hasReleaseSchedule = (quota.releaseSchedule ?? []).length > 0; + const fallbackResetMs = quota.resetAt ? new Date(quota.resetAt).getTime() - nowMs : 0; + const fallbackRemaining = + quota.resetAt && fallbackResetMs <= 0 && quota.usedInWindow > 0 ? limit : quota.remainingInWindow; + const used = Math.max(0, hasReleaseSchedule ? scheduledUsed : limit - Math.max(0, fallbackRemaining)); + const remaining = Math.max(0, Math.min(limit, limit - used)); + const availablePercent = Math.max(0, Math.min(100, Math.round((remaining / limit) * 100))); + const nextReleaseMs = + activeReleaseItems + .map((item) => new Date(item.releasesAt).getTime()) + .filter((releasesAtMs) => Number.isFinite(releasesAtMs) && releasesAtMs > nowMs) + .sort((left, right) => left - right)[0] ?? null; + const resetMs = nextReleaseMs ? nextReleaseMs - nowMs : fallbackResetMs; + const resetLabel = nextReleaseMs || quota.resetAt + ? resetMs > 0 + ? `восст. ${formatShortDuration(resetMs)}` + : "окно свободно" + : "окно свободно"; + const tone = remaining <= 0 ? "blocked" : remaining <= Math.ceil(limit * 0.15) ? "tight" : "ok"; + + useEffect(() => { + const intervalId = window.setInterval(() => setNowMs(Date.now()), 5_000); + + return () => window.clearInterval(intervalId); + }, []); + + return ( +
+
+ Wordstat + + {remaining}/{limit} + +
+ +
+ {availablePercent}% доступно + {used} использ. + {resetLabel} +
+
+ ); +} + function getAnchorReviewStateLabel(state: AnchorReviewContract["state"]) { const labels = { approved: "зафиксировано", @@ -8515,6 +8621,7 @@ export function App() { const [reviewingStrategyQualityProjectId, setReviewingStrategyQualityProjectId] = useState(null); const [reviewingKeywordCleaningProjectId, setReviewingKeywordCleaningProjectId] = useState(null); const [persistingKeywordMapProjectId, setPersistingKeywordMapProjectId] = useState(null); + const [persistingKeywordMapMode, setPersistingKeywordMapMode] = useState<"save" | "strategy" | null>(null); const [materializingTargetKey, setMaterializingTargetKey] = useState(null); const [buildingPatchProjectId, setBuildingPatchProjectId] = useState(null); const [runningPatchDryRunKey, setRunningPatchDryRunKey] = useState(null); @@ -10932,6 +11039,7 @@ export function App() { } ) { setPersistingKeywordMapProjectId(project.id); + setPersistingKeywordMapMode(input?.jumpToStrategy ? "strategy" : "save"); setProjectActionError(null); setProjectActionMessage(null); setProjectActionProjectId(project.id); @@ -11000,6 +11108,7 @@ export function App() { setProjectActionError(getErrorMessage(error, "Не удалось сохранить keyword decisions.")); } finally { setPersistingKeywordMapProjectId(null); + setPersistingKeywordMapMode(null); } } @@ -12425,6 +12534,29 @@ export function App() { ? activeContextReviewRaw : null; const activeMarketEnrichment = activeProject ? (marketEnrichments[activeProject.id] ?? null) : null; + const activeQuotaProjectId = activeProject?.id ?? null; + + useEffect(() => { + if (activeStageIndex !== 4 || !activeQuotaProjectId) { + return; + } + + const intervalId = window.setInterval(() => { + void fetchLatestMarketEnrichment(activeQuotaProjectId) + .then((result) => { + setMarketEnrichments((currentEnrichments) => ({ + ...currentEnrichments, + [activeQuotaProjectId]: result.marketEnrichment + })); + }) + .catch(() => { + // Quota refresh is informational; action handlers surface real failures. + }); + }, 60_000); + + return () => window.clearInterval(intervalId); + }, [activeQuotaProjectId, activeStageIndex]); + const activeKeywordMap = activeProject ? (keywordMaps[activeProject.id] ?? null) : null; const activeSeoStrategy = activeProject ? (seoStrategies[activeProject.id] ?? null) : null; const activeStrategySynthesis = activeProject ? (strategySyntheses[activeProject.id] ?? null) : null; @@ -12812,6 +12944,7 @@ export function App() { activeKeywordStrategyPersistableCount > 0 && !isPersistingActiveKeywordMap ); + const isSubmittingActiveKeywordStrategy = isPersistingActiveKeywordMap && persistingKeywordMapMode === "strategy"; const topbarStageActions = activeProject && activeProjectScanSummary && activeStageIndex === 1 ? ( ) : activeKeywordWorkflowStep === "analysis" && activeAnchorReviewReady ? ( @@ -13798,6 +13931,9 @@ export function App() {
{activeProject ? (
+ {activeStageIndex === 4 && activeMarketEnrichment?.wordstat.quota ? ( + + ) : null} {topbarStageActions}
{activeStageIndex === 5 ? ( @@ -14167,6 +14303,8 @@ export function App() { const isReviewingStrategyQuality = reviewingStrategyQualityProjectId === project.id; const isReviewingKeywordCleaning = reviewingKeywordCleaningProjectId === project.id; const isPersistingKeywordMap = persistingKeywordMapProjectId === project.id; + const isSavingKeywordMap = isPersistingKeywordMap && persistingKeywordMapMode === "save"; + const isSubmittingKeywordStrategy = isPersistingKeywordMap && persistingKeywordMapMode === "strategy"; const isBuildingPatch = buildingPatchProjectId === project.id; const isRunningPatchDryRun = patchArtifact ? runningPatchDryRunKey === `${project.id}:${patchArtifact.changesetId}` @@ -14301,7 +14439,7 @@ export function App() { { detail: seoStrategy.opportunities[0]?.targetPath ?? - currentStrategyNarrative?.landingPlan.find((landing) => landing.targetPath)?.targetPath ?? + currentStrategyNarrative?.landingPlan?.find((landing) => landing.targetPath)?.targetPath ?? "посадочная ещё не выбрана", label: "Страницы", tone: seoStrategy.opportunities.length > 0 ? ("good" as const) : ("warning" as const), @@ -17133,6 +17271,7 @@ export function App() {
Этап 2 · Анализ выбранных ключей Wordstat и внутренняя нормализация показывают, какие фразы подтверждены спросом. + Прогон: {formatDate(marketEnrichment.generatedAt)}
@@ -17484,10 +17623,17 @@ export function App() { {keywordMap && keywordMap.items.length > 0 ? ( handleKeywordReconfigure(project.id)} onRoleChange={(itemId, role) => handleKeywordCurationRoleChange(project.id, itemId, role)} + onSave={() => + void handleApproveKeywordMapDecisions(project, { + roleOverrides: getKeywordCurationRoleOverrides(keywordMap, keywordCurationOverrides) + }) + } onSubmit={() => void handleApproveKeywordMapDecisions(project, { jumpToStrategy: true, @@ -17857,7 +18003,7 @@ export function App() {
- {seoStrategy ? ( + {seoStrategy && seoStrategy.state !== "blocked" ? ( <>
@@ -19536,6 +19682,12 @@ export function App() {
+ ) : seoStrategy ? ( +
+ + Стратегия ещё не сформирована: вернись на этап “Ключи и спрос”, сохрани распределение и нажми + “Формировать стратегию”. +
) : (
diff --git a/seo_mode/seo_mode/app/src/api.ts b/seo_mode/seo_mode/app/src/api.ts index 3900e7d..2163591 100644 --- a/seo_mode/seo_mode/app/src/api.ts +++ b/seo_mode/seo_mode/app/src/api.ts @@ -910,6 +910,27 @@ export type MarketEnrichmentContract = { projectOntologyVersionId: string | null; }>; wordstat: { + quota: { + provider: string; + mode: string; + limitPerHour: number; + limitPerSecond: number; + usedInWindow: number; + remainingInWindow: number; + windowMinutes: number; + windowStartedAt: string | null; + resetAt: string | null; + nextAvailableAt: string; + calculatedAt: string; + utilizationPercent: number; + releaseSchedule: Array<{ + jobId: string; + status: "queued" | "running" | "done" | "failed" | "cancelled"; + requestCount: number; + createdAt: string; + releasesAt: string; + }>; + } | null; latestJob: { id: string; status: "queued" | "running" | "done" | "failed" | "cancelled"; @@ -934,6 +955,10 @@ export type MarketEnrichmentContract = { frequencyGroup: string | null; region: string | null; relatedCount: number; + clusterId: string | null; + clusterTitle: string | null; + priority: "high" | "medium" | "low" | null; + seedSource: "detected" | "ontology" | string | null; collectedAt: string; }>; topResults: Array<{ @@ -943,6 +968,10 @@ export type MarketEnrichmentContract = { frequency: number | null; frequencyGroup: string | null; region: string | null; + sourceClusterId: string | null; + sourceClusterTitle: string | null; + sourcePriority: "high" | "medium" | "low" | null; + sourceSeedSource: "detected" | "ontology" | string | null; }>; summary: { collectedSeedCount: number; diff --git a/seo_mode/seo_mode/app/src/styles.css b/seo_mode/seo_mode/app/src/styles.css index b84a55a..c20b907 100644 --- a/seo_mode/seo_mode/app/src/styles.css +++ b/seo_mode/seo_mode/app/src/styles.css @@ -2919,6 +2919,41 @@ input { overflow-wrap: anywhere; } +.keyword-stage-heading .keyword-stage-run-date { + color: var(--ink); +} + +.keyword-stage-actions { + display: inline-flex; + flex: 0 0 auto; + align-items: center; + justify-content: flex-end; + gap: 0.5rem; +} + +.keyword-stage-icon-action { + display: grid; + width: 3rem; + height: 3rem; + place-items: center; + border: 0; + border-radius: 999px; + background: rgba(255, 255, 255, 0.9); + color: rgba(10, 12, 14, 0.84); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.94), 0 10px 30px rgba(24, 32, 29, 0.08); + cursor: pointer; +} + +.keyword-stage-icon-action:hover { + background: rgba(24, 24, 24, 0.94); + color: #fff; +} + +.keyword-stage-icon-action:disabled { + cursor: not-allowed; + opacity: 0.52; +} + .anchor-review-workspace { background: #fff; } @@ -2958,13 +2993,14 @@ input { } .keyword-curation-board { + --keyword-curation-radius: var(--launcher-radius-card, 1.35rem); display: grid; gap: 12px; min-width: 0; padding: 1rem; background: #fff; border: 0; - border-radius: var(--launcher-radius-card, 1.35rem); + border-radius: var(--keyword-curation-radius); box-shadow: 0 14px 42px rgba(24, 32, 29, 0.055); } @@ -2987,9 +3023,9 @@ input { display: inline-flex; align-items: center; padding: 0 9px; - color: #2e5a67; - background: rgba(46, 90, 103, 0.08); - border: 1px solid rgba(46, 90, 103, 0.14); + color: rgba(8, 8, 10, 0.78); + background: rgba(8, 8, 10, 0.07); + border: 0; border-radius: 999px; font-size: 11px; font-weight: 900; @@ -3035,15 +3071,15 @@ input { padding: 8px; background: rgba(24, 32, 29, 0.035); border: 1px solid rgba(24, 32, 29, 0.07); - border-radius: 8px; + border-radius: var(--keyword-curation-radius); transition: background-color 140ms ease, border-color 140ms ease; } .keyword-curation-column.over { - background: rgba(46, 90, 103, 0.07); - border-color: rgba(46, 90, 103, 0.22); + background: rgba(8, 8, 10, 0.055); + border-color: rgba(8, 8, 10, 0.18); } .keyword-curation-column-head { @@ -3104,7 +3140,7 @@ input { padding: 10px; background: rgba(255, 255, 255, 0.92); border: 1px solid rgba(24, 32, 29, 0.08); - border-radius: 8px; + border-radius: var(--keyword-curation-radius); box-shadow: 0 10px 24px rgba(24, 32, 29, 0.08); user-select: none; } @@ -3133,7 +3169,7 @@ input { color: var(--muted); background: rgba(255, 255, 255, 0.52); border: 1px dashed rgba(24, 32, 29, 0.14); - border-radius: 8px; + border-radius: var(--keyword-curation-radius); font-size: 12px; font-weight: 820; text-align: center; @@ -14153,6 +14189,7 @@ body:has(.seo-launcher-shell) { /* Work panels: the header is part of the same surface, not a separate card. */ .seo-launcher-shell.project-open .topbar { + position: relative !important; display: grid !important; grid-template-columns: minmax(0, 1fr) auto !important; align-items: center !important; @@ -14214,6 +14251,84 @@ body:has(.seo-launcher-shell) { gap: 0.6rem; } +.wordstat-quota-strip { + --wordstat-quota-left: 100%; + display: grid; + width: clamp(15rem, 24vw, 22rem); + min-width: 13.5rem; + gap: 0.24rem; + color: rgba(8, 8, 10, 0.72); +} + +.seo-stage-5 .wordstat-quota-strip { + position: absolute; + top: 50%; + left: 50%; + z-index: 2; + transform: translate(-50%, -50%); +} + +.wordstat-quota-head, +.wordstat-quota-meta { + display: flex; + min-width: 0; + align-items: center; + justify-content: space-between; + gap: 0.44rem; +} + +.wordstat-quota-head span, +.wordstat-quota-meta span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.wordstat-quota-head span { + color: rgba(8, 8, 10, 0.54); + font-size: 0.68rem; + font-weight: 860; + text-transform: uppercase; +} + +.wordstat-quota-head strong { + color: rgba(8, 8, 10, 0.92); + font-size: 0.78rem; + font-weight: 920; + line-height: 1; +} + +.wordstat-quota-bar { + position: relative; + height: 0.27rem; + overflow: hidden; + border-radius: 999px; + background: rgba(8, 8, 10, 0.08); +} + +.wordstat-quota-bar span { + position: absolute; + inset: 0 auto 0 0; + width: var(--wordstat-quota-left); + border-radius: inherit; + background: #e83e8c; +} + +.wordstat-quota-strip.tight .wordstat-quota-bar span { + background: #e83e8c; +} + +.wordstat-quota-strip.blocked .wordstat-quota-bar span { + background: #e83e8c; +} + +.wordstat-quota-meta { + color: rgba(8, 8, 10, 0.52); + font-size: 0.64rem; + font-weight: 760; + line-height: 1; +} + .seo-topbar-primary-action { display: inline-flex; min-height: 2.65rem; @@ -14267,6 +14382,21 @@ body:has(.seo-launcher-shell) { color: #fff; } +@media (max-width: 1360px) { + .seo-stage-5 .seo-topbar-keyword-actions > button { + width: 3rem; + min-width: 3rem; + padding: 0; + gap: 0; + font-size: 0; + } + + .seo-stage-5 .seo-topbar-keyword-actions > button svg { + width: 1rem; + height: 1rem; + } +} + .seo-topbar-workspace-actions em { display: inline-flex; min-width: 1.34rem; diff --git a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts index 2e68c2d..caea07c 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts @@ -80,6 +80,8 @@ export type KeywordCleaningModelTaskContract = { frequency: number | null; frequencyGroup: string | null; region: string | null; + sourceClusterId: string | null; + sourceClusterTitle: string | null; }>; }; outputSchema: { @@ -198,13 +200,27 @@ type KeywordCleaningRunRow = { type SeedQueueItem = MarketEnrichmentContract["seedQueue"][number]; type WordstatTopResult = MarketEnrichmentContract["wordstat"]["topResults"][number]; +type RelatedSourceSeed = Pick< + SeedQueueItem, + "clusterId" | "clusterTitle" | "normalization" | "phrase" | "priority" | "projectOntologyVersionId" | "source" +>; const HARD_TRASH_PATTERNS = [ /(^|\s)(ваканси[ия]|работа|зарплата|резюме)(\s|$)/i, /(^|\s)(реферат|курсовая|диплом|презентация|скачать|pdf|книга)(\s|$)/i, - /(^|\s)(договор|образец|бланк|закон|гост|оквэд)(\s|$)/i + /(^|\s)(образец|бланк)\s+договор/i, + /(^|\s)(закон|гост|оквэд)(\s|$)/i ]; +const CONSUMER_AI_NOISE_PATTERNS = [ + /(^|\s)(бесплатн[а-яёa-z0-9-]*|онлайн|чат|бот|ответ[а-яёa-z0-9-]*|написать|написание|текст[а-яёa-z0-9-]*|песн[а-яёa-z0-9-]*|музык[а-яёa-z0-9-]*|фильм[а-яёa-z0-9-]*|фото|картинк[а-яёa-z0-9-]*|изображен[а-яёa-z0-9-]*|видео|порно|секс|таро|гороскоп)(\s|$)/i, + /нейросет[а-яёa-z0-9-]*\s+(бесплатн[а-яёa-z0-9-]*|онлайн|для\s+текст[а-яёa-z0-9-]*|для\s+картин[а-яёa-z0-9-]*)/i +]; + +const AI_KEYWORD_PATTERN = /(^|\s)(ai|ии)(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект|нейросет/i; +const B2B_KEYWORD_FACET_PATTERN = + /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|решен|1с|crm|erp|bpm|workflow|digital|twin|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн|предприят|корпоратив/i; + const ARTICLE_PATTERNS = [ /(^|\s)(как|что такое|почему|зачем|когда|пример|виды|этапы|способы|инструкция|гайд)(\s|$)/i, /(^|\s)(выбрать|сравнение|обзор|решение проблемы)(\s|$)/i @@ -549,7 +565,11 @@ function enforceKeywordCleaningSemanticGuard( }); } -function getTargetPath(seed: SeedQueueItem, briefPhraseMap: Map, briefClusterMap: Map) { +function getTargetPath( + seed: Pick, + briefPhraseMap: Map, + briefClusterMap: Map +) { const directTarget = briefPhraseMap.get(normalizePhrase(seed.phrase)); if (directTarget) { @@ -656,7 +676,7 @@ function buildSeedItem( function buildRelatedItem( result: WordstatTopResult, - sourceSeed: SeedQueueItem, + sourceSeed: RelatedSourceSeed, index: number, briefPhraseMap: Map, briefClusterMap: Map @@ -697,6 +717,25 @@ function buildRelatedItem( }; } +function buildRelatedSourceSeedFromWordstatResult( + result: WordstatTopResult, + projectOntologyVersionId: string | null +): RelatedSourceSeed | null { + if (!result.sourceClusterId) { + return null; + } + + return { + clusterId: result.sourceClusterId, + clusterTitle: result.sourceClusterTitle ?? result.sourcePhrase ?? result.phrase, + normalization: null, + phrase: result.sourcePhrase ?? result.phrase, + priority: result.sourcePriority ?? "medium", + projectOntologyVersionId, + source: result.sourceSeedSource === "ontology" ? "ontology" : "detected" + }; +} + function getBaseItems(contract: MarketEnrichmentContract) { const briefPhraseMap = buildBriefPhraseMap(contract); const briefClusterMap = buildBriefClusterMap(contract); @@ -715,7 +754,9 @@ function getBaseItems(contract: MarketEnrichmentContract) { } const sourcePhrase = result.sourcePhrase ? normalizePhrase(result.sourcePhrase) : normalizedResult; - const sourceSeed = seedByPhrase.get(sourcePhrase); + const sourceSeed = + seedByPhrase.get(sourcePhrase) ?? + buildRelatedSourceSeedFromWordstatResult(result, contract.projectOntologyVersion?.id ?? null); if (!sourceSeed) { continue; @@ -832,11 +873,33 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key id: result.id, phrase: result.phrase, region: result.region, + sourceClusterId: result.sourceClusterId, + sourceClusterTitle: result.sourceClusterTitle, sourcePhrase: result.sourcePhrase })) }; } +function isConsumerAiNoisePhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return AI_KEYWORD_PATTERN.test(normalizedPhrase) && hasPattern(normalizedPhrase, CONSUMER_AI_NOISE_PATTERNS); +} + +function hasB2BKeywordFacet(phrase: string) { + return B2B_KEYWORD_FACET_PATTERN.test(normalizePhrase(phrase)); +} + +function isBroadAiKeywordNoise(phrase: string | null) { + if (!phrase) { + return false; + } + + const normalizedPhrase = normalizePhrase(phrase); + + return AI_KEYWORD_PATTERN.test(normalizedPhrase) && !hasB2BKeywordFacet(normalizedPhrase); +} + function getClassification(item: KeywordCleaningItem, contextTokens: Set) { const blockers: string[] = []; const exactEvidence = item.frequency !== null; @@ -851,6 +914,9 @@ function getClassification(item: KeywordCleaningItem, contextTokens: Set const isTrash = item.marketRole === "exclude" || hasPattern(item.normalizedPhrase, HARD_TRASH_PATTERNS) || + isConsumerAiNoisePhrase(item.normalizedPhrase) || + isBroadAiKeywordNoise(item.normalizedPhrase) || + (item.source === "wordstat_related" && isBroadAiKeywordNoise(item.sourcePhrase)) || (item.source === "wordstat_related" && !hasContextOverlap(item.phrase, contextTokens)) || (item.source === "wordstat_related" && wordCount(item.phrase) <= 1); @@ -1037,6 +1103,7 @@ function buildKeywordCleaningModelTask( stopConditions: [ "Не отправлять trash в keyword map. Risky/article exact-фразы можно только предсортировать в Stage 3, но не считать approved без пользователя.", "Не придумывать спрос без Wordstat/SERP evidence.", + "Отбрасывать бытовой AI/нейросетевой шум: онлайн, бесплатно, тексты, песни, фото, видео, чат, порно и похожие consumer-запросы не являются B2B SEO-кандидатами без source evidence.", "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря или доминантный смысл approved anchors.", "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.", "Не сохранять rewrite/apply decisions." diff --git a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts index 8186fac..2ccee51 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts @@ -467,7 +467,13 @@ function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleanin const targetCounts = new Map(); const semanticFacetGuard = buildSemanticFacetGuard(contract); const sourceItems = cleaning.items - .filter((item) => item.decision !== "trash") + .filter( + (item) => + item.decision !== "trash" && + item.evidenceStatus === "collected" && + item.frequency !== null && + Boolean(item.wordstatResultId) + ) .sort((left, right) => { const leftRouteWeight = canModelRouteKeyword(left, semanticFacetGuard) ? 1 : 0; const rightRouteWeight = canModelRouteKeyword(right, semanticFacetGuard) ? 1 : 0; diff --git a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts index e132727..d549b95 100644 --- a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts +++ b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts @@ -226,6 +226,184 @@ function normalizePhrase(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } +const EXPANSION_STOP_PARTS = new Set([ + "ai", + "ии", + "искусственный интеллект", + "api", + "bpm", + "crm", + "engine", + "erp", + "hub", + "mcp", + "ops", + "3d", + "demo", + "pilot", + "workspace", + "on-prem", + "private cloud", + "данных", + "данные", + "доступы", + "интеграции", + "модули", + "платформа", + "процессы", + "система" +]); + +const ALLOWED_EXPANSION_LATIN_TOKENS = new Set(["1c", "ai", "api", "bim", "bpm", "crm", "erp", "iot", "workflow"]); + +const EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = [ + [/digital\s+twin|цифров\w*\s+двойн/i, ["цифровой двойник", "цифровые двойники", "digital twin"]], + [/\bbim\b|(^|\s)бим(\s|$)/i, ["bim", "бим"]], + [/1c|1с/i, ["1с", "1c"]], + [/закуп/i, ["закупки", "автоматизация закупок", "управление закупками"]], + [/тендер/i, ["тендеры", "тендерный помощник", "автоматизация тендеров"]], + [/юрид/i, ["юридические процессы", "юридический ассистент", "автоматизация договоров"]], + [/беспилот/i, ["беспилотная техника", "управление беспилотниками", "платформа для беспилотников"]], + [/\biot\b|интернет\s+вещ/i, ["iot", "интернет вещей", "iot платформа"]], + [/телеметр/i, ["телеметрия", "система телеметрии", "платформа телеметрии"]], + [/строител/i, ["строительство", "автоматизация строительства", "строительный контроль"]], + [/эксплуатац/i, ["эксплуатация объектов", "управление эксплуатацией", "автоматизация эксплуатации"]] +]; + +const EXPANSION_QUERY_BLOCKLIST = [ + /(^|\s)(базов[а-яёa-z0-9-]*|трендов[а-яёa-z0-9-]*|коммерческ[а-яёa-z0-9-]*)\s+интент/i, + /(^|\s)интент(:|\s|$)/i, + /(^|\s)(задач|проверок)\.$/i, + /(^|\s)(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)\s+(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)$/i, + /^(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)\s+(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)(\s|$)/i, + /[.!?:]/, + /\b[a-z]+\s+[a-z]+\s+[a-z]+\b/i +]; + +function wordCount(value: string) { + return normalizePhrase(value).split(/\s+/).filter(Boolean).length; +} + +function unique(values: string[]) { + const seen = new Set(); + const result: string[] = []; + + for (const value of values) { + const cleanValue = normalizePhrase(value); + + if (!cleanValue || seen.has(cleanValue)) { + continue; + } + + seen.add(cleanValue); + result.push(cleanValue); + } + + return result; +} + +function splitExpansionParts(value: string) { + return normalizePhrase(value) + .replace(/[()]/g, " ") + .replace(/\s+(?:и|and)\s+/gi, ",") + .split(/\s*(?:,|\/|&|\+|;)\s*/i) + .map((part) => part.trim()) + .filter((part) => wordCount(part) >= 2 || /^[a-z0-9-]{2,16}$/i.test(part)) + .filter((part) => !EXPANSION_STOP_PARTS.has(part)); +} + +function getExpansionSynonyms(value: string) { + return EXPANSION_SYNONYMS.flatMap(([pattern, replacements]) => (pattern.test(value) ? replacements : [])); +} + +function hasSpecificExpansionFacet(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|1с|crm|erp|bpm|workflow|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн/i.test( + normalizedPhrase + ); +} + +function hasUnsafeExpansionLatinToken(phrase: string) { + return phrase + .split(/\s+/) + .flatMap((token) => token.match(/[a-z0-9-]+/gi) ?? []) + .some((token) => !ALLOWED_EXPANSION_LATIN_TOKENS.has(token.toLowerCase())); +} + +function buildExpansionPhraseVariants(phrase: string, contextText: string) { + const variants = new Set(); + const cleanPhrase = normalizePhrase(phrase); + const hasAiContext = /\bai\b|(^|\s)ии(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(contextText); + + if (!cleanPhrase || EXPANSION_STOP_PARTS.has(cleanPhrase)) { + return []; + } + + variants.add(cleanPhrase); + + if (!/платформ/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`платформа ${cleanPhrase}`); + variants.add(`${cleanPhrase} платформа`); + } + + if (!/систем/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`система ${cleanPhrase}`); + } + + if (!/автоматизац/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`автоматизация ${cleanPhrase}`); + } + + if (hasAiContext && !/\bai\b|(^|\s)ии(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`${cleanPhrase} с искусственным интеллектом`); + variants.add(`ai ${cleanPhrase}`); + } + + return [...variants].filter(isSafeWordstatExpansionPhrase); +} + +function isSafeWordstatExpansionPhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + wordCount(normalizedPhrase) >= 2 && + wordCount(normalizedPhrase) <= 7 && + /[а-яё]/i.test(normalizedPhrase) && + normalizedPhrase !== "искусственный интеллект" && + !/[{}[\]<>#$%^*=~`]/.test(normalizedPhrase) && + !EXPANSION_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedPhrase)) && + !hasUnsafeExpansionLatinToken(normalizedPhrase) && + (!/\b(ai|ии)\b|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) || + hasSpecificExpansionFacet(normalizedPhrase)) && + !/(^|\s)(система|платформа)\s+(система|платформа)(\s|$)/i.test(normalizedPhrase) && + !/искусственн[а-яёa-z0-9-]*\s+интеллект.*искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) && + !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]{1,2}$/i.test(token) && !/^(ai|bim)$/i.test(token)) + ); +} + +function buildClusterExpansionPhrases(cluster: SemanticAnalysisRun["clusters"][number], seedPhrase: string) { + const contextText = [ + seedPhrase, + cluster.title, + cluster.targetIntent, + ...cluster.matchedTerms, + ...cluster.queryExamples, + ...cluster.sourceRefs.map((ref) => ref.title), + ...cluster.evidenceSnippets.slice(0, 8).flatMap((snippet) => [snippet.title, snippet.text]), + ...cluster.targetPages + ].join(" "); + const baseParts = unique([ + ...splitExpansionParts(seedPhrase), + ...splitExpansionParts(cluster.title), + ...splitExpansionParts(cluster.targetIntent), + ...cluster.matchedTerms.flatMap(splitExpansionParts), + ...getExpansionSynonyms(contextText) + ]); + + return unique(baseParts.flatMap((part) => buildExpansionPhraseVariants(part, contextText))).slice(0, 16); +} + function buildWordstatSeedMap(wordstat: WordstatEvidence) { return new Map(wordstat.seedResults.map((result) => [normalizePhrase(result.phrase), result])); } @@ -271,18 +449,18 @@ function getSeedEvidenceStatus( return "collected"; } - if (wordstat.latestJob?.status === "failed") { - return "collection_failed"; - } - - if (wordstat.latestJob?.status === "done" && wordstatResult && wordstatResult.relatedCount > 0) { + if (wordstatResult && wordstatResult.relatedCount > 0) { return "collected_related_only"; } - if (wordstat.latestJob?.status === "done" && wordstatResult) { + if (wordstatResult) { return "collected_no_signal"; } + if (wordstat.latestJob?.status === "failed") { + return "collection_failed"; + } + return getEvidenceStatus(providers, ["wordstat"]); } @@ -330,6 +508,95 @@ function buildSeedQueue( }); } +function shouldExpandSeed(seed: MarketEnrichmentContract["seedQueue"][number]) { + return ( + seed.frequency === null && + (seed.evidenceStatus === "collected_no_signal" || + seed.evidenceStatus === "collected_related_only" || + seed.evidenceStatus === "collection_failed") + ); +} + +function buildWordstatExpansionSeeds( + analysis: SemanticAnalysisRun, + market: MarketEnrichmentContract +): WordstatSeedCandidate[] { + const clusterById = new Map(analysis.clusters.map((cluster) => [cluster.id, cluster])); + const existingPhrases = new Set([ + ...market.seedQueue.map((seed) => normalizePhrase(seed.phrase)), + ...market.wordstat.seedResults.map((result) => normalizePhrase(result.phrase)), + ...market.wordstat.topResults.map((result) => normalizePhrase(result.phrase)) + ]); + const candidatesByCluster = new Map(); + const clusterOrder: string[] = []; + + for (const seed of market.seedQueue.filter(shouldExpandSeed)) { + const cluster = clusterById.get(seed.clusterId); + + if (!cluster) { + continue; + } + + for (const phrase of buildClusterExpansionPhrases(cluster, seed.phrase)) { + const normalizedPhrase = normalizePhrase(phrase); + + if (existingPhrases.has(normalizedPhrase)) { + continue; + } + + existingPhrases.add(normalizedPhrase); + + if (!candidatesByCluster.has(seed.clusterId)) { + candidatesByCluster.set(seed.clusterId, []); + clusterOrder.push(seed.clusterId); + } + + const clusterCandidates = candidatesByCluster.get(seed.clusterId); + + if (!clusterCandidates || clusterCandidates.length >= 8) { + continue; + } + + clusterCandidates.push({ + clusterId: seed.clusterId, + clusterTitle: seed.clusterTitle, + confidence: Math.max(0.42, Math.min((seed.normalization?.confidence ?? 0.6) * 0.9, 0.82)), + phrase: normalizedPhrase, + priority: seed.priority, + reason: `Expansion seed для Wordstat: исходная фраза "${seed.phrase}" не дала exact demand; аналог собран из semantic cluster/source evidence.`, + source: seed.source + }); + } + } + + const diversifiedCandidates: WordstatSeedCandidate[] = []; + + for (let index = 0; diversifiedCandidates.length < 48; index += 1) { + let added = false; + + for (const clusterId of clusterOrder) { + const candidate = candidatesByCluster.get(clusterId)?.[index]; + + if (!candidate) { + continue; + } + + diversifiedCandidates.push(candidate); + added = true; + + if (diversifiedCandidates.length >= 48) { + break; + } + } + + if (!added) { + break; + } + } + + return diversifiedCandidates; +} + function buildBriefEvidence( analysis: SemanticAnalysisRun, providers: MarketProviderDescriptor[], @@ -473,8 +740,7 @@ export async function getMarketEnrichmentContract(projectId: string): Promise seed.phrase); - const wordstat = await getLatestWordstatEvidence(projectId, analysis?.runId ?? null, approvedWordstatPhrases); + const wordstat = await getLatestWordstatEvidence(projectId, analysis?.runId ?? null); const seedQueue = analysis ? buildSeedQueue(anchorReview, providers, wordstat, projectOntologyVersion) : []; const briefEvidence = analysis ? buildBriefEvidence(analysis, providers, projectOntologyVersion) : []; const configuredProviderCount = providers.filter((provider) => provider.status === "connected").length; @@ -546,6 +812,12 @@ export async function runMarketEnrichment(projectId: string): Promise 0) { + await collectWordstatEvidence(projectId, analysis, [...normalizedSeeds, ...expansionSeeds]); + } } catch (error) { console.error(error); } diff --git a/seo_mode/seo_mode/server/src/market/wordstatProvider.ts b/seo_mode/seo_mode/server/src/market/wordstatProvider.ts index 05a3f9b..084a4c9 100644 --- a/seo_mode/seo_mode/server/src/market/wordstatProvider.ts +++ b/seo_mode/seo_mode/server/src/market/wordstatProvider.ts @@ -61,6 +61,8 @@ export interface WordstatProvider { collect(request: WordstatCollectRequest): Promise; } +const YANDEX_WORDSTAT_MIN_REQUEST_INTERVAL_MS = 125; + export class WordstatProviderUnavailableError extends Error { constructor(message: string) { super(message); @@ -68,6 +70,12 @@ export class WordstatProviderUnavailableError extends Error { } } +function delay(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + function getFrequencyGroup(frequency: number | null, explicitGroup?: unknown) { if (typeof explicitGroup === "string" && explicitGroup.trim()) { return explicitGroup.trim(); @@ -317,7 +325,11 @@ class YandexApiWordstatProvider implements WordstatProvider { this.config.authType === "iam_token" ? `Bearer ${this.config.apiToken}` : `Api-Key ${this.config.apiToken}`; const seedResponses = []; - for (const seed of request.seeds) { + for (const [index, seed] of request.seeds.entries()) { + if (index > 0) { + await delay(YANDEX_WORDSTAT_MIN_REQUEST_INTERVAL_MS); + } + const payload = await postJson( endpoint, { diff --git a/seo_mode/seo_mode/server/src/market/wordstatRepository.ts b/seo_mode/seo_mode/server/src/market/wordstatRepository.ts index 5694cea..b9181a0 100644 --- a/seo_mode/seo_mode/server/src/market/wordstatRepository.ts +++ b/seo_mode/seo_mode/server/src/market/wordstatRepository.ts @@ -49,6 +49,10 @@ type WordstatResultRow = { id: string; job_id: string; seed_id: string | null; + seed_cluster_id: string | null; + seed_cluster_title: string | null; + seed_priority: "high" | "medium" | "low" | null; + seed_source: "detected" | "ontology" | string | null; phrase: string; source_phrase: string | null; normalized_phrase: string | null; @@ -60,7 +64,33 @@ type WordstatResultRow = { created_at: Date; }; +const WORDSTAT_STATS_REQUESTS_PER_HOUR_LIMIT = 100; +const WORDSTAT_STATS_REQUESTS_PER_SECOND_LIMIT = 10; + +export type WordstatQuotaSnapshot = { + provider: string; + mode: string; + limitPerHour: number; + limitPerSecond: number; + usedInWindow: number; + remainingInWindow: number; + windowMinutes: number; + windowStartedAt: string | null; + resetAt: string | null; + nextAvailableAt: string; + calculatedAt: string; + utilizationPercent: number; + releaseSchedule: Array<{ + jobId: string; + status: WordstatJobRow["status"]; + requestCount: number; + createdAt: string; + releasesAt: string; + }>; +}; + export type WordstatEvidence = { + quota: WordstatQuotaSnapshot | null; latestJob: { id: string; status: WordstatJobRow["status"]; @@ -85,6 +115,10 @@ export type WordstatEvidence = { frequencyGroup: string | null; region: string | null; relatedCount: number; + clusterId: string | null; + clusterTitle: string | null; + priority: "high" | "medium" | "low" | null; + seedSource: "detected" | "ontology" | string | null; collectedAt: string; }>; topResults: Array<{ @@ -94,6 +128,10 @@ export type WordstatEvidence = { frequency: number | null; frequencyGroup: string | null; region: string | null; + sourceClusterId: string | null; + sourceClusterTitle: string | null; + sourcePriority: "high" | "medium" | "low" | null; + sourceSeedSource: "detected" | "ontology" | string | null; }>; summary: { collectedSeedCount: number; @@ -405,7 +443,11 @@ function getLatestUniqueWordstatResults(results: WordstatResultRow[]) { return Array.from(resultByIdentity.values()).sort(compareWordstatResultRows); } -function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]): WordstatEvidence { +function buildEvidence( + job: WordstatJobRow | null, + results: WordstatResultRow[], + quota: WordstatQuotaSnapshot | null +): WordstatEvidence { const seedRows = results.filter((result) => result.source_phrase === result.phrase || !result.source_phrase); const relatedCounts = new Map(); @@ -416,6 +458,7 @@ function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]) } return { + quota, latestJob: job ? mapJob(job) : null, seedResults: seedRows.map((result) => ({ id: result.id, @@ -426,6 +469,10 @@ function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]) frequencyGroup: result.frequency_group, region: result.region, relatedCount: relatedCounts.get(result.phrase) ?? 0, + clusterId: result.seed_cluster_id, + clusterTitle: result.seed_cluster_title, + priority: result.seed_priority, + seedSource: result.seed_source, collectedAt: result.created_at.toISOString() })), topResults: getRepresentativeTopResultRows(results) @@ -435,7 +482,11 @@ function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]) sourcePhrase: result.source_phrase, frequency: result.frequency, frequencyGroup: result.frequency_group, - region: result.region + region: result.region, + sourceClusterId: result.seed_cluster_id, + sourceClusterTitle: result.seed_cluster_title, + sourcePriority: result.seed_priority, + sourceSeedSource: result.seed_source })), summary: { collectedSeedCount: seedRows.length, @@ -458,7 +509,7 @@ function buildEvidence(job: WordstatJobRow | null, results: WordstatResultRow[]) async function ensureSeedRows(projectId: string, analysis: SemanticAnalysisRun, seedCandidates: WordstatSeedCandidate[]) { const seedRows: SeedRow[] = []; - for (const seed of seedCandidates.slice(0, 80)) { + for (const seed of seedCandidates) { const inserted = await pool.query( ` insert into seeds ( @@ -548,6 +599,89 @@ async function getCollectedSeedPhraseSet(projectId: string, semanticRunId: strin return new Set(result.rows.map((row) => normalizePhrase(row.normalized_phrase ?? row.phrase))); } +async function getWordstatQuotaSnapshot(provider: string, mode: string): Promise { + const result = await pool.query<{ + id: string; + status: WordstatJobRow["status"]; + created_at: Date; + request_count: string | number | null; + }>( + ` + select + id, + status, + created_at, + jsonb_array_length(requested_phrases) as request_count + from wordstat_jobs + where provider = $1 + and mode = $2 + and status in ('running', 'done', 'failed') + and created_at >= now() - interval '1 hour' + and jsonb_array_length(requested_phrases) > 0 + order by created_at asc; + `, + [provider, mode] + ); + const releaseSchedule = result.rows.map((row) => { + const createdAt = row.created_at; + const releasesAt = new Date(createdAt.getTime() + 60 * 60 * 1000); + + return { + createdAt: createdAt.toISOString(), + jobId: row.id, + releasesAt: releasesAt.toISOString(), + requestCount: Number(row.request_count ?? 0), + status: row.status + }; + }); + const usedCount = releaseSchedule.reduce((sum, item) => sum + item.requestCount, 0); + const remaining = Math.max(0, WORDSTAT_STATS_REQUESTS_PER_HOUR_LIMIT - usedCount); + const calculatedAt = new Date().toISOString(); + const resetAt = releaseSchedule[0]?.releasesAt ?? null; + + return { + calculatedAt, + limitPerHour: WORDSTAT_STATS_REQUESTS_PER_HOUR_LIMIT, + limitPerSecond: WORDSTAT_STATS_REQUESTS_PER_SECOND_LIMIT, + mode, + nextAvailableAt: remaining > 0 ? calculatedAt : (resetAt ?? calculatedAt), + provider, + releaseSchedule, + remainingInWindow: remaining, + resetAt, + usedInWindow: usedCount, + utilizationPercent: Math.min(100, Math.round((usedCount / WORDSTAT_STATS_REQUESTS_PER_HOUR_LIMIT) * 100)), + windowMinutes: 60, + windowStartedAt: releaseSchedule[0]?.createdAt ?? null + }; +} + +async function getWordstatQuotaSnapshotForEvidence(job: WordstatJobRow | null) { + if (job) { + return getWordstatQuotaSnapshot(job.provider, job.mode); + } + + const provider = await createWordstatProvider(); + const status = provider.getStatus(); + + if (status.provider === "disabled") { + return null; + } + + return getWordstatQuotaSnapshot(status.provider, status.mode); +} + +async function getWordstatHourlyBudget(provider: string, mode: string) { + const snapshot = await getWordstatQuotaSnapshot(provider, mode); + + return { + limit: snapshot.limitPerHour, + remaining: snapshot.remainingInWindow, + resetAt: snapshot.nextAvailableAt, + usedCount: snapshot.usedInWindow + }; +} + async function createJob( projectId: string, semanticRunId: string, @@ -737,16 +871,24 @@ export async function collectWordstatEvidence( const seedRows = await ensureSeedRows(projectId, analysis, seedCandidates); const collectedSeedPhrases = await getCollectedSeedPhraseSet(projectId, analysis.runId); - const seeds = seedRows - .filter((row) => !collectedSeedPhrases.has(normalizePhrase(row.phrase))) - .slice(0, status.maxSeedsPerRun) - .map(toWordstatSeed); - const region = "ru"; + const uncollectedSeedRows = seedRows.filter((row) => !collectedSeedPhrases.has(normalizePhrase(row.phrase))); - if (seeds.length === 0) { + if (uncollectedSeedRows.length === 0) { return getLatestWordstatEvidence(projectId, analysis.runId); } + const budget = await getWordstatHourlyBudget(status.provider, status.mode); + const maxSeedsForRun = Math.min(status.maxSeedsPerRun, budget.remaining); + + if (maxSeedsForRun <= 0) { + throw new Error( + `Wordstat hourly quota exhausted: ${budget.usedCount}/${budget.limit} statistics requests used. Retry after ${budget.resetAt}.` + ); + } + + const seeds = uncollectedSeedRows.slice(0, maxSeedsForRun).map(toWordstatSeed); + const region = "ru"; + const job = await createJob(projectId, analysis.runId, status.provider, status.mode, region, seeds); if (!job) { @@ -778,7 +920,7 @@ export async function getLatestWordstatEvidence( scopePhrases?: string[] ): Promise { if (!semanticRunId) { - return buildEvidence(null, []); + return buildEvidence(null, [], await getWordstatQuotaSnapshotForEvidence(null)); } const jobResult = await pool.query( @@ -801,15 +943,18 @@ export async function getLatestWordstatEvidence( updated_at from wordstat_jobs where project_id = $1 and semantic_run_id = $2 - order by created_at desc + order by + case when status = 'done' then 0 else 1 end, + created_at desc limit 1; `, [projectId, semanticRunId] ); const latestJob = jobResult.rows[0] ?? null; + const quota = await getWordstatQuotaSnapshotForEvidence(latestJob); if (!latestJob) { - return buildEvidence(null, []); + return buildEvidence(null, [], quota); } const results = await pool.query( @@ -818,6 +963,10 @@ export async function getLatestWordstatEvidence( wr.id, wr.job_id, wr.seed_id, + s.cluster_id as seed_cluster_id, + s.cluster_title as seed_cluster_title, + s.priority as seed_priority, + s.source as seed_source, wr.phrase, wr.source_phrase, wr.normalized_phrase, @@ -829,6 +978,7 @@ export async function getLatestWordstatEvidence( wr.created_at from wordstat_results wr join wordstat_jobs wj on wj.id = wr.job_id + left join seeds s on s.id = wr.seed_id where wj.project_id = $1 and wj.semantic_run_id = $2 and wj.status = 'done' @@ -839,7 +989,7 @@ export async function getLatestWordstatEvidence( const latestUniqueResults = getLatestUniqueWordstatResults(results.rows); if (!scopePhrases || scopePhrases.length === 0) { - return buildEvidence(latestJob, latestUniqueResults); + return buildEvidence(latestJob, latestUniqueResults, quota); } const scopePhraseSet = new Set(scopePhrases.map(normalizePhrase)); @@ -855,5 +1005,5 @@ export async function getLatestWordstatEvidence( result_count: scopedResults.length }; - return buildEvidence(scopedJob, scopedResults); + return buildEvidence(scopedJob, scopedResults, quota); } diff --git a/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts b/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts index 53ee0e2..bfae6a7 100644 --- a/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts +++ b/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts @@ -158,7 +158,7 @@ function getOutputTokenBudget(task: SeoModelTaskContract | null) { } if (task.taskType === "seo.keyword_cleaning") { - return 2400; + return 5200; } if (task.taskType === "seo.serp_interpretation") { diff --git a/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts b/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts index e01c227..e026c25 100644 --- a/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts +++ b/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts @@ -87,6 +87,18 @@ export type SeoNormalizationModelTaskContract = { queryExamples: string[]; evidenceLevel: "none" | "thin" | "moderate" | "strong"; }>; + verticalOpportunities: Array<{ + id: string; + title: string; + priority: "high" | "medium" | "low"; + status: "covered" | "weak" | "missing"; + targetIntent: string; + matchedTerms: string[]; + queryExamples: string[]; + sourceTitles: string[]; + sourceSnippets: string[]; + targetPages: string[]; + }>; sourceSeeds: Array<{ phrase: string; clusterId: string; @@ -100,6 +112,7 @@ export type SeoNormalizationModelTaskContract = { | "normalize_market_queries" | "reject_noise" | "mark_human_review" + | "expand_vertical_queries" | "route_wordstat" | "route_serp" >; @@ -207,6 +220,9 @@ export type SeoNormalizationContract = { const GENERIC_SINGLE_WORDS = new Set([ "ai", + "ии", + "искусственный интеллект", + "engine", "bpm", "workflow", "автоматизация", @@ -317,12 +333,13 @@ function normalizeSeoNormalizationOutput( throw new Error("SEO normalization output должен иметь intentGroups, seedStrategy и rejectedSeeds."); } - const wordstatApprovedCount = output.seedStrategy.filter((seed) => seed.review.sendToWordstat).length; - const serpApprovedCount = output.seedStrategy.filter((seed) => seed.review.sendToSerp).length; - const needsHumanReviewCount = output.seedStrategy.filter((seed) => seed.review.status === "needs_human").length; - const autoApprovedCount = output.seedStrategy.filter((seed) => seed.review.status === "auto_approved").length; + const seedStrategy = output.seedStrategy.map(normalizeModelSeedReview); + const wordstatApprovedCount = seedStrategy.filter((seed) => seed.review.sendToWordstat).length; + const serpApprovedCount = seedStrategy.filter((seed) => seed.review.sendToSerp).length; + const needsHumanReviewCount = seedStrategy.filter((seed) => seed.review.status === "needs_human").length; + const autoApprovedCount = seedStrategy.filter((seed) => seed.review.status === "auto_approved").length; const rejectedSeedCount = - output.rejectedSeeds.length + output.seedStrategy.filter((seed) => seed.review.status === "rejected").length; + output.rejectedSeeds.length + seedStrategy.filter((seed) => seed.review.status === "rejected").length; return { ...output, @@ -339,13 +356,45 @@ function normalizeSeoNormalizationOutput( autoApprovedCount, intentGroupCount: output.intentGroups.length, needsHumanReviewCount, - normalizedSeedCount: output.seedStrategy.length, + normalizedSeedCount: seedStrategy.length, rejectedSeedCount, serpApprovedCount, wordstatApprovedCount, wordstatQueryCount: wordstatApprovedCount }, - state: output.seedStrategy.length > 0 ? "ready" : "not_ready" + seedStrategy, + state: seedStrategy.length > 0 ? "ready" : "not_ready" + }; +} + +function isSourceGroundedVerticalSeed(seed: SeoNormalizationSeed) { + return ( + seed.source === "detected" && + seed.marketRole !== "exclude" && + !isLowValuePhrase(seed.phrase) && + !isBroadSinglePhrase(seed.phrase) && + wordCount(seed.phrase) >= 2 && + seed.confidence >= 0.5 + ); +} + +function normalizeModelSeedReview(seed: SeoNormalizationSeed): SeoNormalizationSeed { + if (seed.review.status !== "rejected" || !isSourceGroundedVerticalSeed(seed)) { + return seed; + } + + return { + ...seed, + needsHumanReview: true, + review: { + ...seed.review, + blockers: Array.from(new Set([...seed.review.blockers, "source_grounded_vertical_review"])), + reason: + `${seed.review.reason || "Model rejected source-grounded vertical."} Backend guard перевёл source-grounded vertical в needs_human: модель не должна молча выбрасывать потенциальную вертикаль сайта.`, + sendToSerp: false, + sendToWordstat: false, + status: "needs_human" + } }; } @@ -503,8 +552,171 @@ function collectClusterPhrases( ...(contextHint?.sourcePhrases ?? []), ...(contextHint?.marketAngles ?? []).filter((angle) => wordCount(angle) <= 6) ]; + const expansionPhrases = buildClusterExpansionPhrases(cluster, contextHint); - return unique([...seeds.map((seed) => seed.phrase), ...cluster.queryExamples, ...briefPhrases, ...safeHintPhrases]); + return unique([ + ...seeds.map((seed) => seed.phrase), + ...cluster.queryExamples, + ...briefPhrases, + ...safeHintPhrases, + ...expansionPhrases + ]); +} + +const MARKET_EXPANSION_STOP_PARTS = new Set([ + "ai", + "bpm", + "crm", + "erp", + "hub", + "ops", + "api", + "mcp", + "3d", + "demo", + "pilot", + "workspace", + "on-prem", + "private cloud", + "данных", + "данные", + "доступы", + "интеграции", + "модули", + "платформа", + "процессы", + "система" +]); + +const ALLOWED_MARKET_EXPANSION_LATIN_TOKENS = new Set(["1c", "ai", "api", "bim", "bpm", "crm", "erp", "iot", "workflow"]); + +const MARKET_EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = [ + [/digital\s+twin|цифров\w*\s+двойн/i, ["цифровой двойник", "цифровые двойники", "digital twin"]], + [/\bbim\b|(^|\s)бим(\s|$)/i, ["bim", "бим"]], + [/1c|1с/i, ["1с", "1c"]], + [/закуп/i, ["закупки", "автоматизация закупок", "управление закупками"]], + [/тендер/i, ["тендеры", "тендерный помощник", "автоматизация тендеров"]], + [/юрид/i, ["юридические процессы", "юридический ассистент", "автоматизация договоров"]], + [/беспилот/i, ["беспилотная техника", "управление беспилотниками", "платформа для беспилотников"]], + [/\biot\b|интернет\s+вещ/i, ["iot", "интернет вещей", "iot платформа"]], + [/телеметр/i, ["телеметрия", "система телеметрии", "платформа телеметрии"]], + [/строител/i, ["строительство", "автоматизация строительства", "строительный контроль"]], + [/эксплуатац/i, ["эксплуатация объектов", "управление эксплуатацией", "автоматизация эксплуатации"]] +]; + +const MARKET_EXPANSION_QUERY_BLOCKLIST = [ + /(^|\s)(базов[а-яёa-z0-9-]*|трендов[а-яёa-z0-9-]*|коммерческ[а-яёa-z0-9-]*)\s+интент/i, + /(^|\s)интент(:|\s|$)/i, + /(^|\s)(задач|проверок)\.$/i, + /(^|\s)(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)\s+(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)$/i, + /^(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)\s+(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)(\s|$)/i, + /[.!?:]/, + /\b[a-z]+\s+[a-z]+\s+[a-z]+\b/i +]; + +function splitMarketExpansionParts(value: string) { + return normalizePhrase(value) + .replace(/[()]/g, " ") + .replace(/\s+(?:и|and)\s+/gi, ",") + .split(/\s*(?:,|\/|&|\+|;)\s*/i) + .map((part) => part.trim()) + .filter((part) => wordCount(part) >= 2 || /^[a-z0-9-]{2,16}$/i.test(part)) + .filter((part) => !MARKET_EXPANSION_STOP_PARTS.has(part)); +} + +function getMarketExpansionSynonyms(value: string) { + return MARKET_EXPANSION_SYNONYMS.flatMap(([pattern, replacements]) => (pattern.test(value) ? replacements : [])); +} + +function hasSpecificMarketExpansionFacet(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|1с|crm|erp|bpm|workflow|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн/i.test( + normalizedPhrase + ); +} + +function hasUnsafeMarketExpansionLatinToken(phrase: string) { + return phrase + .split(/\s+/) + .flatMap((token) => token.match(/[a-z0-9-]+/gi) ?? []) + .some((token) => !ALLOWED_MARKET_EXPANSION_LATIN_TOKENS.has(token.toLowerCase())); +} + +function buildPhraseExpansionVariants(phrase: string, clusterText: string) { + const variants = new Set(); + const cleanPhrase = normalizePhrase(phrase); + const hasAiContext = /\bai\b|(^|\s)ии(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(clusterText); + + if (!cleanPhrase || MARKET_EXPANSION_STOP_PARTS.has(cleanPhrase)) { + return []; + } + + variants.add(cleanPhrase); + + if (!/платформ/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`платформа ${cleanPhrase}`); + variants.add(`${cleanPhrase} платформа`); + } + + if (!/систем/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`система ${cleanPhrase}`); + } + + if (!/автоматизац/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`автоматизация ${cleanPhrase}`); + } + + if (hasAiContext && !/\bai\b|(^|\s)ии(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + variants.add(`${cleanPhrase} с искусственным интеллектом`); + variants.add(`ai ${cleanPhrase}`); + } + + return [...variants].filter(isSafeWordstatExpansionPhrase); +} + +function isSafeWordstatExpansionPhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + wordCount(normalizedPhrase) >= 2 && + wordCount(normalizedPhrase) <= 7 && + /[а-яё]/i.test(normalizedPhrase) && + normalizedPhrase !== "искусственный интеллект" && + !/[{}[\]<>#$%^*=~`]/.test(normalizedPhrase) && + !MARKET_EXPANSION_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedPhrase)) && + !hasUnsafeMarketExpansionLatinToken(normalizedPhrase) && + (!/\b(ai|ии)\b|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) || + hasSpecificMarketExpansionFacet(normalizedPhrase)) && + !/(^|\s)(система|платформа)\s+(система|платформа)(\s|$)/i.test(normalizedPhrase) && + !/искусственн[а-яёa-z0-9-]*\s+интеллект.*искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) && + !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]{1,2}$/i.test(token) && !/^(ai|bim)$/i.test(token)) + ); +} + +function buildClusterExpansionPhrases( + cluster: SemanticAnalysisRun["clusters"][number], + contextHint: SeoContextReviewRun["normalizationHints"][number] | undefined +) { + const clusterText = [ + cluster.title, + cluster.targetIntent, + ...cluster.matchedTerms, + ...cluster.queryExamples, + ...(contextHint?.sourcePhrases ?? []), + ...(contextHint?.marketAngles ?? []), + ...cluster.evidenceSnippets.slice(0, 8).flatMap((snippet) => [snippet.title, snippet.text]), + ...cluster.sourceRefs.map((ref) => ref.title) + ].join(" "); + const baseParts = unique([ + ...splitMarketExpansionParts(cluster.title), + ...splitMarketExpansionParts(cluster.targetIntent), + ...cluster.matchedTerms.flatMap(splitMarketExpansionParts), + ...(contextHint?.marketAngles ?? []).flatMap(splitMarketExpansionParts), + ...getMarketExpansionSynonyms(clusterText) + ]); + + return unique(baseParts.flatMap((part) => buildPhraseExpansionVariants(part, clusterText))).slice(0, 18); } function buildGroupQueries(phrases: string[], clusterTitle: string, siteTopTerms: string[], stopPhrases: string[]) { @@ -685,6 +897,21 @@ function buildModelTaskContract( targetIntent: cluster.targetIntent, title: cluster.title })), + verticalOpportunities: analysis.clusters + .filter((cluster) => cluster.status !== "missing" || cluster.priority === "high") + .slice(0, 24) + .map((cluster) => ({ + id: cluster.id, + matchedTerms: cluster.matchedTerms.slice(0, 16), + priority: cluster.priority, + queryExamples: cluster.queryExamples.slice(0, 12), + sourceTitles: cluster.sourceRefs.slice(0, 8).map((ref) => ref.title), + sourceSnippets: cluster.evidenceSnippets.slice(0, 8).map((snippet) => snippet.text), + status: cluster.status, + targetIntent: cluster.targetIntent, + targetPages: cluster.targetPages.slice(0, 8), + title: cluster.title + })), sourceSeeds: analysis.seedCandidates.slice(0, 80).map((seed) => ({ clusterId: seed.clusterId, phrase: seed.phrase, @@ -703,6 +930,7 @@ function buildModelTaskContract( "normalize_market_queries", "reject_noise", "mark_human_review", + "expand_vertical_queries", "route_wordstat", "route_serp" ], @@ -728,6 +956,8 @@ function buildModelTaskContract( stopConditions: [ "Недостаточно source evidence для отделения бизнеса от технической оболочки сайта.", "Фраза выглядит как соседний рынок или business expansion без явного user approval.", + "Не отбрасывать source-grounded vertical opportunity: если точной рыночной формулировки нет, вернуть needs_human seed или expansion query, а не rejected.", + "Для каждой видимой продуктовой/отраслевой вертикали предложить 2-6 Wordstat-safe аналогов: коротких, рыночных, без внутренних labels.", "Нельзя определить, отправлять ли фразу в Wordstat/SERP без ручного review." ] };