diff --git a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts index 0eb5a33..db0f6a7 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts @@ -89,11 +89,28 @@ export type KeywordCleaningModelTaskContract = { decisionValues: KeywordCleaningDecision[]; }; decisionPolicy: { + broadFrequencyDoesNotOverrideSemanticFit: boolean; + demoteRelatedWhenDominantFacetMissing: boolean; + preserveApprovedAnchorFacets: boolean; useRequiresExactEvidence: boolean; riskyWhenNoPageBinding: boolean; trashWhenOffContext: boolean; articleDoesNotEnterRewrite: boolean; }; + semanticGuard: { + approvedAnchorCount: number; + dominantProtectedFacets: Array<{ + approvedAnchorCount: number; + approvedAnchorShare: number; + examples: string[]; + stem: string; + }>; + policy: { + broadFrequencyDoesNotOverrideSemanticFit: boolean; + demoteRelatedWhenDominantFacetMissing: boolean; + preserveSourceAnchorFacetsForRelated: boolean; + }; + }; stopConditions: string[]; }; @@ -208,6 +225,110 @@ function wordCount(value: string) { return normalizePhrase(value).split(/\s+/).filter(Boolean).length; } +const KEYWORD_CLEANING_FACET_STOP_WORDS = new Set([ + "a", + "an", + "and", + "by", + "for", + "in", + "of", + "on", + "or", + "the", + "to", + "без", + "в", + "для", + "и", + "или", + "как", + "на", + "о", + "об", + "от", + "по", + "при", + "с", + "со", + "что", + "это" +]); + +const GENERIC_KEYWORD_CLEANING_FACET_STEMS = new Set([ + "автоматизац", + "автоматизаци", + "бизнес", + "данн", + "задач", + "инструмент", + "компан", + "контур", + "модул", + "платформ", + "приложен", + "процесс", + "проект", + "разработк", + "решен", + "сервис", + "систем", + "сайт", + "управлен", + "услуг" +]); + +function stemKeywordCleaningToken(token: string) { + const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е"); + + if (/^(ai|ии)$/i.test(normalizedToken) || /^(artificial|intelligence)$/i.test(normalizedToken)) { + return "ai"; + } + + if (/^(искусствен|интеллект)/i.test(normalizedToken)) { + return "ai"; + } + + if (/^[a-z0-9]+$/i.test(normalizedToken)) { + return normalizedToken.toLowerCase(); + } + + return normalizedToken.replace( + /(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i, + "" + ); +} + +function getKeywordCleaningFacetStems(value: string) { + return new Set( + normalizePhrase(value) + .replace(/ё/g, "е") + .split(/[^a-zа-я0-9]+/i) + .map((token) => token.trim()) + .filter((token) => token.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(token)) + .map(stemKeywordCleaningToken) + .filter((stem) => stem.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(stem)) + ); +} + +function isProtectedKeywordCleaningFacetStem( + stem: string, + approvedAnchorCount: number, + stemDocumentCounts: Map +) { + if (GENERIC_KEYWORD_CLEANING_FACET_STEMS.has(stem)) { + return false; + } + + if (/^[a-z0-9]+$/i.test(stem)) { + return true; + } + + const documentShare = approvedAnchorCount > 0 ? (stemDocumentCounts.get(stem) ?? 0) / approvedAnchorCount : 0; + + return stem.length >= 7 || documentShare >= 0.18; +} + function hasPattern(value: string, patterns: RegExp[]) { return patterns.some((pattern) => pattern.test(value)); } @@ -300,6 +421,134 @@ function buildBriefClusterMap(contract: MarketEnrichmentContract) { return clusterMap; } +type KeywordCleaningFacetGuard = { + approvedAnchorCount: number; + dominantProtectedStems: Array<{ count: number; stem: string }>; + examplesByStem: Map; + stemDocumentCounts: Map; +}; + +function buildKeywordCleaningFacetGuard(contract: MarketEnrichmentContract): KeywordCleaningFacetGuard { + const stemDocumentCounts = new Map(); + const examplesByStem = new Map(); + const approvedAnchors = contract.anchorReview.wordstatQueue.slice(0, 120); + + for (const anchor of approvedAnchors) { + for (const stem of getKeywordCleaningFacetStems(`${anchor.phrase} ${anchor.clusterTitle}`)) { + stemDocumentCounts.set(stem, (stemDocumentCounts.get(stem) ?? 0) + 1); + + const examples = examplesByStem.get(stem) ?? []; + + if (examples.length < 3 && !examples.map(normalizePhrase).includes(normalizePhrase(anchor.phrase))) { + examples.push(anchor.phrase); + examplesByStem.set(stem, examples); + } + } + } + + const dominantProtectedStems = [...stemDocumentCounts.entries()] + .filter(([stem, count]) => { + const share = approvedAnchors.length > 0 ? count / approvedAnchors.length : 0; + + return share >= 0.3 && isProtectedKeywordCleaningFacetStem(stem, approvedAnchors.length, stemDocumentCounts); + }) + .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0], "ru")) + .slice(0, 12) + .map(([stem, count]) => ({ count, stem })); + + return { + approvedAnchorCount: approvedAnchors.length, + dominantProtectedStems, + examplesByStem, + stemDocumentCounts + }; +} + +function buildKeywordCleaningSemanticGuard(contract: MarketEnrichmentContract): KeywordCleaningModelTaskContract["semanticGuard"] { + const guard = buildKeywordCleaningFacetGuard(contract); + const dominantProtectedFacets = guard.dominantProtectedStems + .map((facet) => ({ + approvedAnchorCount: facet.count, + approvedAnchorShare: Number((facet.count / Math.max(guard.approvedAnchorCount, 1)).toFixed(3)), + examples: guard.examplesByStem.get(facet.stem) ?? [], + stem: facet.stem + })); + + return { + approvedAnchorCount: guard.approvedAnchorCount, + dominantProtectedFacets, + policy: { + broadFrequencyDoesNotOverrideSemanticFit: true, + demoteRelatedWhenDominantFacetMissing: true, + preserveSourceAnchorFacetsForRelated: true + } + }; +} + +function getProtectedSourceFacetStems(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) { + if (item.source !== "wordstat_related" || !item.sourcePhrase) { + return new Set(); + } + + const sourceStems = getKeywordCleaningFacetStems(`${item.sourcePhrase} ${item.clusterTitle}`); + + return new Set( + [...sourceStems].filter((stem) => + isProtectedKeywordCleaningFacetStem(stem, guard.approvedAnchorCount, guard.stemDocumentCounts) + ) + ); +} + +function getKeywordCleaningSemanticDowngradeReason(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) { + const phraseStems = getKeywordCleaningFacetStems(item.phrase); + const sourceProtectedStems = getProtectedSourceFacetStems(item, guard); + const missingSourceStems = [...sourceProtectedStems].filter((stem) => !phraseStems.has(stem)); + + if (sourceProtectedStems.size > 0 && missingSourceStems.length === sourceProtectedStems.size) { + return `Wordstat related потерял protected-признак исходного якоря (${missingSourceStems.join(", ")}).`; + } + + const dominantProtectedStems = guard.dominantProtectedStems.map((item) => item.stem); + const missingDominantStems = dominantProtectedStems.filter((stem) => !phraseStems.has(stem)); + + if (dominantProtectedStems.length > 0 && missingDominantStems.length === dominantProtectedStems.length) { + return `Фраза не содержит доминантный protected-смысл approved anchors (${missingDominantStems.join(", ")}).`; + } + + return null; +} + +function enforceKeywordCleaningSemanticGuard( + market: MarketEnrichmentContract, + items: KeywordCleaningItem[] +): KeywordCleaningItem[] { + const guard = buildKeywordCleaningFacetGuard(market); + + if (guard.approvedAnchorCount === 0) { + return items; + } + + return items.map((item) => { + if (item.decision !== "use" && item.decision !== "support") { + return item; + } + + const semanticDowngradeReason = getKeywordCleaningSemanticDowngradeReason(item, guard); + + if (!semanticDowngradeReason) { + return item; + } + + return { + ...item, + blockers: unique([...item.blockers, "semantic_facet_loss"]), + confidence: Math.min(item.confidence, 0.62), + decision: "risky", + reason: `${semanticDowngradeReason} Backend guard понизил фразу в risky: частотность не перекрывает потерю смысла.` + }; + }); +} + function getTargetPath(seed: SeedQueueItem, briefPhraseMap: Map, briefClusterMap: Map) { const directTarget = briefPhraseMap.get(normalizePhrase(seed.phrase)); @@ -704,7 +953,11 @@ function buildReadiness(items: KeywordCleaningItem[], lanes: KeywordCleaningCont articleCount: lanes.article.length, exactEvidenceCount: items.filter((item) => item.frequency !== null).length, keywordMapCandidateCount: items.filter( - (item) => item.decision !== "trash" && item.evidenceStatus === "collected" && item.frequency !== null && item.targetPath + (item) => + (item.decision === "use" || item.decision === "support") && + item.evidenceStatus === "collected" && + item.frequency !== null && + item.targetPath ).length, missingPageBindingCount: items.filter((item) => item.targetPath === null && item.decision !== "trash").length, riskyCount: lanes.risky.length, @@ -734,6 +987,9 @@ function buildKeywordCleaningModelTask( ], decisionPolicy: { articleDoesNotEnterRewrite: true, + broadFrequencyDoesNotOverrideSemanticFit: true, + demoteRelatedWhenDominantFacetMissing: true, + preserveApprovedAnchorFacets: true, riskyWhenNoPageBinding: true, trashWhenOffContext: true, useRequiresExactEvidence: true @@ -777,9 +1033,11 @@ function buildKeywordCleaningModelTask( providerMode: "model_provider_contract", schemaVersion: "seo-keyword-cleaning-task.v1", semanticRunId: contract.semanticRunId, + semanticGuard: buildKeywordCleaningSemanticGuard(contract), stopConditions: [ "Не отправлять trash в keyword map. Risky/article exact-фразы можно только предсортировать в Stage 3, но не считать approved без пользователя.", "Не придумывать спрос без Wordstat/SERP evidence.", + "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря или доминантный смысл approved anchors.", "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.", "Не сохранять rewrite/apply decisions." ], @@ -824,7 +1082,7 @@ function buildNextActions(contract: Omit function buildFallbackContract(projectId: string, market: MarketEnrichmentContract): KeywordCleaningContract { const modelTask = buildKeywordCleaningModelTask(projectId, market); - const items = classifyItems(market, getBaseItems(market)); + const items = enforceKeywordCleaningSemanticGuard(market, classifyItems(market, getBaseItems(market))); const lanes = buildLanes(items); const readiness = buildReadiness(items, lanes); const contractWithoutActions: Omit = { @@ -990,13 +1248,34 @@ export async function getKeywordCleaningContract(projectId: string): Promise 0 + ? `Очищено ${items.length} фраз: ${readiness.useCount} use, ${readiness.supportCount} support, ${readiness.articleCount} article, ${readiness.riskyCount} risky, ${readiness.trashCount} trash.` + : "Нет фраз для keyword cleaning." + }; } export async function saveKeywordCleaningManual( @@ -1051,7 +1330,7 @@ export async function saveKeywordCleaningFromModelProvider( const alignedOutput: KeywordCleaningContract = { ...output, - items: mergeModelProviderItemsWithMarketEvidence(output.items, market), + items: enforceKeywordCleaningSemanticGuard(market, mergeModelProviderItemsWithMarketEvidence(output.items, market)), modelTask, projectOntologyVersion: market.projectOntologyVersion, semanticRunId: market.semanticRunId, diff --git a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts index e0fe430..8ac38fb 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts @@ -103,30 +103,6 @@ function getKeywordProfileText(item: Pick) { - const text = getKeywordProfileText(item); - - return /(^|\s)(ai|ии)(\s|$)/i.test(text) || /(агент|агентн|ассистент|assistant|agentic)/i.test(text); -} - -function hasProductLikeHybridSignal(value: string) { - const text = normalizePhrase(value); - const hasProductNoun = /(систем[ауы]?|платформ[ауы]?|сервис|software|product|suite)/i.test(text); - const hasProcessNoun = /(бизнес[-\s]?процесс|bpms?|workflow|процесс[а-яё]*)/i.test(text); - - return hasProductNoun && hasProcessNoun; -} - -function isBroadArticleBacklogPhrase(value: string) { - const text = normalizePhrase(value); - - return /^автоматизаци[яи]\s+бизнес[-\s]?процесс(ов|ы)?$/i.test(text); -} - -function isHybridSecondaryCandidate(item: KeywordCleaningItem) { - return item.frequency !== null && hasProductLikeHybridSignal(item.phrase) && !isAiDifferentiatorCandidate(item); -} - function isReviewedOntology(version: MarketEnrichmentContract["projectOntologyVersion"]) { return ( Boolean(version) && @@ -214,6 +190,17 @@ type SemanticFacetGuard = { stemDocumentCounts: Map; }; +type SemanticFitAssessment = { + label: "lost_project_facet" | "lost_source_facet" | "strong" | "weak"; + matchedProjectStems: string[]; + matchedSourceStems: string[]; + missingProjectStems: string[]; + missingSourceStems: string[]; + projectCoverage: number | null; + score: number; + sourceCoverage: number | null; +}; + function stemKeywordToken(token: string) { const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е"); @@ -318,14 +305,62 @@ function hasSemanticFacetLoss(item: KeywordCleaningItem, guard: SemanticFacetGua return getMissingSemanticFacetStems(item, guard).length > 0; } -function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) { - if (guard.dominantProtectedStems.size === 0) { - return false; +function getCoverage(matchedCount: number, totalCount: number) { + return totalCount > 0 ? matchedCount / totalCount : null; +} + +function getSemanticFitAssessment(item: KeywordCleaningItem, guard: SemanticFacetGuard): SemanticFitAssessment { + const phraseStems = getKeywordFacetStems(item.phrase); + const sourceStems = item.source === "wordstat_related" ? getProtectedSourceFacetStems(item, guard) : new Set(); + const projectStems = guard.dominantProtectedStems; + const matchedSourceStems = [...sourceStems].filter((stem) => phraseStems.has(stem)); + const matchedProjectStems = [...projectStems].filter((stem) => phraseStems.has(stem)); + const missingSourceStems = [...sourceStems].filter((stem) => !phraseStems.has(stem)); + const missingProjectStems = [...projectStems].filter((stem) => !phraseStems.has(stem)); + const sourceCoverage = getCoverage(matchedSourceStems.length, sourceStems.size); + const projectCoverage = getCoverage(matchedProjectStems.length, projectStems.size); + const score = Math.min(sourceCoverage ?? 1, projectCoverage ?? 1); + + if (sourceCoverage === 0 && sourceStems.size > 0) { + return { + label: "lost_source_facet", + matchedProjectStems, + matchedSourceStems, + missingProjectStems, + missingSourceStems, + projectCoverage, + score, + sourceCoverage + }; } - const phraseStems = getKeywordFacetStems(item.phrase); + if (projectCoverage === 0 && projectStems.size > 0) { + return { + label: "lost_project_facet", + matchedProjectStems, + matchedSourceStems, + missingProjectStems, + missingSourceStems, + projectCoverage, + score, + sourceCoverage + }; + } - return [...guard.dominantProtectedStems].every((stem) => !phraseStems.has(stem)); + return { + label: score >= 0.5 ? "strong" : "weak", + matchedProjectStems, + matchedSourceStems, + missingProjectStems, + missingSourceStems, + projectCoverage, + score, + sourceCoverage + }; +} + +function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) { + return getSemanticFitAssessment(item, guard).label === "lost_project_facet"; } function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) { @@ -335,13 +370,16 @@ function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) { } function canModelRouteKeyword(item: KeywordCleaningItem, guard?: SemanticFacetGuard) { + const semanticFit = guard ? getSemanticFitAssessment(item, guard) : null; + return ( + (item.decision === "use" || item.decision === "support") && item.evidenceStatus === "collected" && item.frequency !== null && Boolean(item.projectOntologyVersionId) && Boolean(item.wordstatResultId) && Boolean(item.targetPath) && - (!guard || (!hasSemanticFacetLoss(item, guard) && !missesDominantProjectFacet(item, guard))) + (!semanticFit || (semanticFit.label === "strong" && semanticFit.score >= 0.5)) ); } @@ -374,14 +412,14 @@ function getKeywordRole(item: KeywordCleaningItem, indexInTarget: number, guard: return "support"; } - if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") { - return "secondary"; - } - if (isClarifyingKeywordCandidate(item)) { return "differentiator"; } + if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") { + return "secondary"; + } + if (item.priority === "high" || item.priority === "medium") { return "secondary"; } @@ -439,6 +477,13 @@ function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleanin return rightPriorityWeight - leftPriorityWeight; } + const leftSemanticFit = getSemanticFitAssessment(left, semanticFacetGuard).score; + const rightSemanticFit = getSemanticFitAssessment(right, semanticFacetGuard).score; + + if (leftSemanticFit !== rightSemanticFit) { + return rightSemanticFit - leftSemanticFit; + } + const leftFrequency = left.frequency ?? -1; const rightFrequency = right.frequency ?? -1; @@ -514,15 +559,15 @@ function getKeywordMapItemReason( } if (role === "validate" && hasSemanticFacetLoss(item, semanticFacetGuard)) { - return "Wordstat related потерял доминантный признак исходного якоря: оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта."; + const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard); + + return `Wordstat related потерял protected-признак исходного якоря (${semanticFit.missingSourceStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта.`; } if (role === "validate" && missesDominantProjectFacet(item, semanticFacetGuard)) { - return "Фраза не содержит доминантный смысл из подтверждённой очереди якорей: оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически."; - } + const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard); - if (isBroadArticleBacklogPhrase(item.phrase) && role === "validate") { - return "Широкая head-фраза уходит в article/backlog: не делаем её прямой основной ставкой текущей посадочной без отдельного SERP page-type решения."; + return `Фраза не содержит доминантный protected-смысл из подтверждённой очереди (${semanticFit.missingProjectStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически.`; } if (item.decision === "article") {