diff --git a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts index e20b581..196b801 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts @@ -84,6 +84,17 @@ export type KeywordCleaningModelTaskContract = { sourceClusterId: string | null; sourceClusterTitle: string | null; }>; + sourceGroupCritics: Array<{ + sourcePhrase: string; + verdict: "clean" | "mixed" | "source_drift"; + reason: string; + resultCount: number; + exactCount: number; + driftSignalCount: number; + strictFitSignalCount: number; + driftExamples: string[]; + strictFitExamples: string[]; + }>; }; outputSchema: { schemaVersion: "keyword-cleaning.v1"; @@ -95,6 +106,8 @@ export type KeywordCleaningModelTaskContract = { broadFrequencyDoesNotOverrideSemanticFit: boolean; demoteRelatedWhenDominantFacetMissing: boolean; preserveApprovedAnchorFacets: boolean; + quarantineNoisyRelatedGroups: boolean; + sourceGroupDriftDoesNotEnterKeywordMap: boolean; useRequiresExactEvidence: boolean; riskyWhenNoPageBinding: boolean; trashWhenOffContext: boolean; @@ -205,6 +218,22 @@ type RelatedSourceSeed = Pick< SeedQueueItem, "clusterId" | "clusterTitle" | "normalization" | "phrase" | "priority" | "projectOntologyVersionId" | "source" >; +type WordstatSourceGroupCriticVerdict = "clean" | "mixed" | "source_drift"; +type WordstatSourceGroupCritic = { + sourcePhrase: string; + normalizedSourcePhrase: string; + verdict: WordstatSourceGroupCriticVerdict; + reason: string; + resultCount: number; + exactCount: number; + signalCount: number; + driftSignalCount: number; + strictFitSignalCount: number; + sourceHasStrictFit: boolean; + sourceLooksWeak: boolean; + driftExamples: string[]; + strictFitExamples: string[]; +}; const HARD_TRASH_PATTERNS = [ /(^|\s)(ваканси[ия]|работа|зарплата|резюме)(\s|$)/i, @@ -249,6 +278,26 @@ const PRODUCT_TARIFF_QUALIFIER_PATTERN = const AI_KEYWORD_PATTERN = /(^|\s)(ai|ии)(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект|нейросет/i; const B2B_KEYWORD_FACET_PATTERN = /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|решен|платформ|стоимост|внедрен|купить|заказать|тариф|1с|crm|erp|bpm|workflow|digital|twin|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн|предприят|корпоратив/i; +const SOURCE_GROUP_CRITIC_MIN_RESULTS = 4; +const SOURCE_GROUP_CRITIC_MODEL_LIMIT = 24; +const SOURCE_GROUP_CRITIC_DRIFT_PATTERNS = [ + /(^|\s)(звук[а-яёa-z0-9-]*|слог[а-яёa-z0-9-]*|букв[а-яёa-z0-9-]*|реч[а-яёa-z0-9-]*|логопед[а-яёa-z0-9-]*|артикуляц[а-яёa-z0-9-]*|дошкольн[а-яёa-z0-9-]*|произношен[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(основн[а-яёa-z0-9-]*|базов[а-яёa-z0-9-]*)\s+(поняти[а-яёa-z0-9-]*|цель|цели|задач[а-яёa-z0-9-]*|принцип[а-яёa-z0-9-]*|вид[а-яёa-z0-9-]*|тип[а-яёa-z0-9-]*|функци[а-яёa-z0-9-]*|этап[а-яёa-z0-9-]*|направлен[а-яёa-z0-9-]*|уровн[а-яёa-z0-9-]*|средств[а-яёa-z0-9-]*)/i, + /(^|\s)(поняти[ея]|определен[а-яёa-z0-9-]*|цель|цели|задач[аи]?|принцип[а-яёa-z0-9-]*|виды|типы|функци[а-яёa-z0-9-]*|этапы|направлен[а-яёa-z0-9-]*|уровни|схем[а-яёa-z0-9-]*|структур[а-яёa-z0-9-]*|классификац[а-яёa-z0-9-]*|реферат|конспект|презентац[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(упражнен[а-яёa-z0-9-]*|задани[а-яёa-z0-9-]*|игр[а-яёa-z0-9-]*|картинк[а-яёa-z0-9-]*|стих[а-яёa-z0-9-]*)(\s|$)/i +]; +const SOURCE_GROUP_CRITIC_WEAK_SOURCE_PATTERNS = [ + /(^|\s)(предложени[еяюях]*|основн[а-яёa-z0-9-]*|поняти[а-яёa-z0-9-]*|цель|цели|задач[а-яёa-z0-9-]*|принцип[а-яёa-z0-9-]*|вид[а-яёa-z0-9-]*|тип[а-яёa-z0-9-]*|функци[а-яёa-z0-9-]*|этап[а-яёa-z0-9-]*|направлен[а-яёa-z0-9-]*)(\s|$)/i +]; +const SOURCE_GROUP_STRICT_PRODUCT_FIT_PATTERNS = [ + /(^|\s)(агент[а-яёa-z0-9-]*|ассистент[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(ai|ии|llm|ml|нейросет[а-яёa-z0-9-]*|искусственн[а-яёa-z0-9-]*\s+интеллект)\s+(агент[а-яёa-z0-9-]*|ассистент[а-яёa-z0-9-]*|платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*|сервис[а-яёa-z0-9-]*|для\s+(бизнес[а-яёa-z0-9-]*|компан[а-яёa-z0-9-]*|предприят[а-яёa-z0-9-]*|корпоративн[а-яёa-z0-9-]*))/i, + /(^|\s)(платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*|сервис[а-яёa-z0-9-]*)\s+(ai|ии|llm|ml|нейросет[а-яёa-z0-9-]*|искусственн[а-яёa-z0-9-]*\s+интеллект)(\s|$)/i, + /(^|\s)(b2b|enterprise|корпоративн[а-яёa-z0-9-]*|предприят[а-яёa-z0-9-]*|компан[а-яёa-z0-9-]*|бизнес[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(crm|erp|bpm|workflow|api|1с|1c|эдо|документооборот|закуп[а-яёa-z0-9-]*|тендер[а-яёa-z0-9-]*|договор[а-яёa-z0-9-]*|юридическ[а-яёa-z0-9-]*|финанс[а-яёa-z0-9-]*|поручен[а-яёa-z0-9-]*|согласован[а-яёa-z0-9-]*|база\s+знан[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(интеграц[а-яёa-z0-9-]*|внедрен[а-яёa-z0-9-]*|разработк[а-яёa-z0-9-]*|под\s+ключ|заказать|купить|стоимост[а-яёa-z0-9-]*|цен[а-яёa-z0-9-]*|платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*)(\s|$)/i, + /(^|\s)(строител[а-яёa-z0-9-]*|производств[а-яёa-z0-9-]*|промышленн[а-яёa-z0-9-]*|беспилот[а-яёa-z0-9-]*|iot|телеметр[а-яёa-z0-9-]*|digital\s+twin|цифров[а-яёa-z0-9-]*\s+двойн[а-яёa-z0-9-]*|bim|бим)(\s|$)/i +]; const KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT = 60; const KEYWORD_CLEANING_MODEL_LANDING_BRIEF_LIMIT = 18; const KEYWORD_CLEANING_MODEL_NORMALIZATION_GROUP_LIMIT = 24; @@ -712,6 +761,233 @@ function promoteExactDemandItems(market: MarketEnrichmentContract, items: Keywor }); } +function hasProtectedProjectFacet(value: string, guard: KeywordCleaningFacetGuard) { + const phraseStems = getKeywordCleaningFacetStems(value); + + return guard.dominantProtectedStems.some((facet) => phraseStems.has(facet.stem)); +} + +function hasStrictRelatedProductFit(market: MarketEnrichmentContract, value: string, guard: KeywordCleaningFacetGuard) { + const normalizedValue = normalizePhrase(value); + + if (hasProtectedProjectFacet(normalizedValue, guard)) { + return true; + } + + if (hasPattern(normalizedValue, SOURCE_GROUP_STRICT_PRODUCT_FIT_PATTERNS)) { + return true; + } + + return ( + AI_KEYWORD_PATTERN.test(normalizedValue) && + hasB2BKeywordFacet(normalizedValue) && + !isConsumerAiNoisePhrase(normalizedValue) && + !hasEducationIntentMismatch(market, normalizedValue) + ); +} + +function hasSourceGroupRelatedDriftSignal(market: MarketEnrichmentContract, phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + hasPattern(normalizedPhrase, SOURCE_GROUP_CRITIC_DRIFT_PATTERNS) || + isConsumerAiNoisePhrase(normalizedPhrase) || + isOffContextRelatedNoisePhrase(normalizedPhrase) || + hasThirdPartyBrandDemandMismatch(normalizedPhrase) || + hasNonProductMarketIntentMismatch(normalizedPhrase) || + hasEducationIntentMismatch(market, normalizedPhrase) + ); +} + +function sourceGroupLooksWeak(sourcePhrase: string, guard: KeywordCleaningFacetGuard) { + const normalizedSourcePhrase = normalizePhrase(sourcePhrase); + + return hasPattern(normalizedSourcePhrase, SOURCE_GROUP_CRITIC_WEAK_SOURCE_PATTERNS) && !hasProtectedProjectFacet(normalizedSourcePhrase, guard); +} + +function getSourceGroupCriticVerdict(input: { + driftShare: number; + driftSignalCount: number; + signalCount: number; + sourceHasStrictFit: boolean; + sourceLooksWeak: boolean; + strictFitShare: number; + strictFitSignalCount: number; +}): WordstatSourceGroupCriticVerdict { + if (input.signalCount < SOURCE_GROUP_CRITIC_MIN_RESULTS) { + return "clean"; + } + + if ( + input.sourceLooksWeak && + input.driftSignalCount >= 2 && + input.strictFitSignalCount <= 1 && + input.strictFitShare < 0.5 + ) { + return "source_drift"; + } + + if (!input.sourceHasStrictFit && input.driftShare >= 0.45 && input.strictFitShare < 0.45 && input.signalCount >= 6) { + return "source_drift"; + } + + if (input.driftShare >= 0.7 && input.strictFitShare < 0.35 && input.signalCount >= 6) { + return "source_drift"; + } + + if (input.driftShare >= 0.3 && input.strictFitShare < 0.6) { + return "mixed"; + } + + return "clean"; +} + +function buildWordstatSourceGroupCriticReason(critic: Omit) { + if (critic.verdict === "source_drift") { + return `Source-group critic пометил Wordstat related-группу "${critic.sourcePhrase}" как source drift: ${critic.driftSignalCount}/${critic.signalCount} строк ушли в омонимический или справочный спрос, строгий product-fit держат ${critic.strictFitSignalCount}.`; + } + + if (critic.verdict === "mixed") { + return `Source-group critic пометил Wordstat related-группу "${critic.sourcePhrase}" как mixed: в пачке есть drift-сигналы, поэтому related без строгого product-fit нельзя повышать в карту только по частотности.`; + } + + return `Source-group critic не нашел групповой drift у Wordstat related-группы "${critic.sourcePhrase}".`; +} + +function buildWordstatSourceGroupCritics( + market: MarketEnrichmentContract, + items: KeywordCleaningItem[] +): Map { + const guard = buildKeywordCleaningFacetGuard(market); + const groups = new Map(); + + for (const item of items) { + if (item.source !== "wordstat_related" || !item.sourcePhrase) { + continue; + } + + const normalizedSourcePhrase = normalizePhrase(item.sourcePhrase); + const group = groups.get(normalizedSourcePhrase) ?? { items: [], sourcePhrase: item.sourcePhrase }; + group.items.push(item); + groups.set(normalizedSourcePhrase, group); + } + + const critics = new Map(); + + for (const [normalizedSourcePhrase, group] of groups.entries()) { + const exactItems = group.items.filter((item) => item.frequency !== null && Boolean(item.wordstatResultId)); + const signalItems = exactItems.length >= SOURCE_GROUP_CRITIC_MIN_RESULTS ? exactItems : group.items; + const driftItems = signalItems.filter((item) => hasSourceGroupRelatedDriftSignal(market, item.phrase)); + const strictFitItems = signalItems.filter( + (item) => hasStrictRelatedProductFit(market, item.phrase, guard) && !hasSourceGroupRelatedDriftSignal(market, item.phrase) + ); + const signalCount = signalItems.length; + const driftShare = signalCount > 0 ? driftItems.length / signalCount : 0; + const strictFitShare = signalCount > 0 ? strictFitItems.length / signalCount : 0; + const sourceHasStrictFit = hasStrictRelatedProductFit(market, group.sourcePhrase, guard); + const weakSource = sourceGroupLooksWeak(group.sourcePhrase, guard); + const baseCritic = { + driftExamples: driftItems.slice(0, 5).map((item) => item.phrase), + driftSignalCount: driftItems.length, + exactCount: exactItems.length, + normalizedSourcePhrase, + resultCount: group.items.length, + signalCount, + sourceHasStrictFit, + sourceLooksWeak: weakSource, + sourcePhrase: group.sourcePhrase, + strictFitExamples: strictFitItems.slice(0, 5).map((item) => item.phrase), + strictFitSignalCount: strictFitItems.length, + verdict: getSourceGroupCriticVerdict({ + driftShare, + driftSignalCount: driftItems.length, + signalCount, + sourceHasStrictFit, + sourceLooksWeak: weakSource, + strictFitShare, + strictFitSignalCount: strictFitItems.length + }) + }; + const critic: WordstatSourceGroupCritic = { + ...baseCritic, + reason: buildWordstatSourceGroupCriticReason(baseCritic) + }; + + critics.set(normalizedSourcePhrase, critic); + } + + return critics; +} + +function getSourceGroupCriticEvidenceRef(critic: WordstatSourceGroupCritic) { + return `backend:source-group-critic:${slugify(critic.sourcePhrase)}`; +} + +function enforceWordstatSourceGroupCritic( + market: MarketEnrichmentContract, + items: KeywordCleaningItem[] +): KeywordCleaningItem[] { + const guard = buildKeywordCleaningFacetGuard(market); + const critics = buildWordstatSourceGroupCritics(market, items); + + return items.map((item) => { + if (item.source !== "wordstat_related" || !item.sourcePhrase) { + return item; + } + + const critic = critics.get(normalizePhrase(item.sourcePhrase)); + + if (!critic || critic.verdict === "clean") { + return item; + } + + const selfHasDriftSignal = hasSourceGroupRelatedDriftSignal(market, item.phrase); + const selfHasStrictFit = hasStrictRelatedProductFit(market, item.phrase, guard) && !selfHasDriftSignal; + const blocker = critic.verdict === "source_drift" ? "source_group_related_drift" : "source_group_related_mixed"; + const evidenceRefs = unique([...item.evidenceRefs, getSourceGroupCriticEvidenceRef(critic)]); + const blockers = unique([...(Array.isArray(item.blockers) ? item.blockers : []), blocker]); + + if (critic.verdict === "mixed") { + if ((item.decision !== "use" && item.decision !== "support") || selfHasStrictFit) { + return { + ...item, + blockers, + evidenceRefs + }; + } + + return { + ...item, + blockers, + confidence: Math.min(item.confidence, 0.58), + decision: "risky", + evidenceRefs, + reason: `${critic.reason} Backend guard понизил related без строгого product-fit в risky: mixed-группа требует ручной проверки.` + }; + } + + if (selfHasStrictFit) { + return { + ...item, + blockers, + confidence: Math.min(item.confidence, 0.58), + decision: "risky", + evidenceRefs, + reason: `${critic.reason} Фраза сама держит product-fit, поэтому не удаляется, но не может попасть в approved lanes без ручной проверки source-группы.` + }; + } + + return { + ...item, + blockers, + confidence: Math.max(item.confidence, 0.9), + decision: "trash", + evidenceRefs, + reason: `${critic.reason} Backend guard отбросил фразу до keyword map: загрязненная source-группа не должна протаскивать омонимический related-спрос.` + }; + }); +} + function enforceKeywordCleaningNoiseGuard(items: KeywordCleaningItem[]): KeywordCleaningItem[] { return items.map((item) => { if (isOffContextRelatedNoisePhrase(item.normalizedPhrase)) { @@ -756,7 +1032,10 @@ function enforceKeywordCleaningBackendGuards( items: KeywordCleaningItem[] ): KeywordCleaningItem[] { return enforceKeywordCleaningNoiseGuard( - promoteExactDemandItems(market, rescueExactTrashItems(market, enforceKeywordCleaningSemanticGuard(market, items))) + enforceWordstatSourceGroupCritic( + market, + promoteExactDemandItems(market, rescueExactTrashItems(market, enforceKeywordCleaningSemanticGuard(market, items))) + ) ); } @@ -1116,6 +1395,15 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key .sort((left, right) => right.score - left.score || left.index - right.index) .slice(0, KEYWORD_CLEANING_MODEL_TOP_RESULT_LIMIT) .map((item) => item.result); + const modelSourceGroupCritics = [...buildWordstatSourceGroupCritics(contract, getBaseItems(contract)).values()] + .filter((critic) => critic.verdict !== "clean") + .sort( + (left, right) => + (right.verdict === "source_drift" ? 1 : 0) - (left.verdict === "source_drift" ? 1 : 0) || + right.driftSignalCount - left.driftSignalCount || + right.resultCount - left.resultCount + ) + .slice(0, SOURCE_GROUP_CRITIC_MODEL_LIMIT); return { approvedAnchors: contract.anchorReview.wordstatQueue.slice(0, KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT).map((anchor) => ({ @@ -1170,6 +1458,17 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key sourceClusterId: result.sourceClusterId, sourceClusterTitle: truncateTaskText(result.sourceClusterTitle), sourcePhrase: result.sourcePhrase + })), + sourceGroupCritics: modelSourceGroupCritics.map((critic) => ({ + driftExamples: critic.driftExamples.map((phrase) => truncateTaskText(phrase)), + driftSignalCount: critic.driftSignalCount, + exactCount: critic.exactCount, + reason: truncateTaskText(critic.reason, 220), + resultCount: critic.resultCount, + sourcePhrase: truncateTaskText(critic.sourcePhrase), + strictFitExamples: critic.strictFitExamples.map((phrase) => truncateTaskText(phrase)), + strictFitSignalCount: critic.strictFitSignalCount, + verdict: critic.verdict })) }; } @@ -1439,6 +1738,8 @@ function buildKeywordCleaningModelTask( broadFrequencyDoesNotOverrideSemanticFit: true, demoteRelatedWhenDominantFacetMissing: false, preserveApprovedAnchorFacets: true, + quarantineNoisyRelatedGroups: true, + sourceGroupDriftDoesNotEnterKeywordMap: true, riskyWhenNoPageBinding: false, trashWhenOffContext: true, useRequiresExactEvidence: true @@ -1494,6 +1795,7 @@ function buildKeywordCleaningModelTask( "Отбрасывать бытовой AI/нейросетевой шум: онлайн, бесплатно, тексты, песни, фото, видео, чат, порно и похожие consumer-запросы не являются B2B SEO-кандидатами без source evidence.", "Образовательный спрос (курсы, обучение, тренинги, сертификаты) не отправлять в product keyword map, если Product Understanding/normalization не подтверждает образовательный offer сайта.", "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря; глобальный доминантный термин проекта не должен запрещать отдельные подтверждённые ветки спроса.", + "Проверять sourceGroupCritics после Wordstat: source_drift-группы не отправлять в keyword map, mixed-группы повышать только при строгом product/source fit у конкретной фразы.", "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.", "Не сохранять rewrite/apply decisions." ], diff --git a/seo_mode/seo_mode/server/src/scripts/checkDemandCoverage.ts b/seo_mode/seo_mode/server/src/scripts/checkDemandCoverage.ts index de6e986..deb7d53 100644 --- a/seo_mode/seo_mode/server/src/scripts/checkDemandCoverage.ts +++ b/seo_mode/seo_mode/server/src/scripts/checkDemandCoverage.ts @@ -160,6 +160,15 @@ assert.ok( keywordCleaningSource.includes("getKeywordCleaningModelTopResultScore"), "Keyword cleaning model task must keep a slim ranked payload so AI Workspace does not fail with request entity too large." ); +assert.ok( + keywordCleaningSource.includes("buildWordstatSourceGroupCritics") && + keywordCleaningSource.includes("enforceWordstatSourceGroupCritic") && + keywordCleaningSource.includes("source_group_related_drift") && + keywordCleaningSource.includes("sourceGroupCritics") && + keywordCleaningSource.includes("quarantineNoisyRelatedGroups") && + keywordCleaningSource.includes("sourceGroupDriftDoesNotEnterKeywordMap"), + "Keyword cleaning must run a source-group/related-fit critic after Wordstat so homonym related batches are quarantined before keyword map." +); assert.ok( modelProviderRegistrySource.includes('"seo.market_frontier_discovery"'), "Model provider registry must expose seo.market_frontier_discovery as a runnable task type."