Harden Wordstat source group filtering

seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts

@@ -84,6 +84,17 @@ export type KeywordCleaningModelTaskContract = {
       sourceClusterId: string | null;
       sourceClusterTitle: string | null;
     }>;
+    sourceGroupCritics: Array<{
+      sourcePhrase: string;
+      verdict: "clean" | "mixed" | "source_drift";
+      reason: string;
+      resultCount: number;
+      exactCount: number;
+      driftSignalCount: number;
+      strictFitSignalCount: number;
+      driftExamples: string[];
+      strictFitExamples: string[];
+    }>;
   };
   outputSchema: {
     schemaVersion: "keyword-cleaning.v1";
@@ -95,6 +106,8 @@ export type KeywordCleaningModelTaskContract = {
     broadFrequencyDoesNotOverrideSemanticFit: boolean;
     demoteRelatedWhenDominantFacetMissing: boolean;
     preserveApprovedAnchorFacets: boolean;
+    quarantineNoisyRelatedGroups: boolean;
+    sourceGroupDriftDoesNotEnterKeywordMap: boolean;
     useRequiresExactEvidence: boolean;
     riskyWhenNoPageBinding: boolean;
     trashWhenOffContext: boolean;
@@ -205,6 +218,22 @@ type RelatedSourceSeed = Pick<
   SeedQueueItem,
   "clusterId" | "clusterTitle" | "normalization" | "phrase" | "priority" | "projectOntologyVersionId" | "source"
 >;
+type WordstatSourceGroupCriticVerdict = "clean" | "mixed" | "source_drift";
+type WordstatSourceGroupCritic = {
+  sourcePhrase: string;
+  normalizedSourcePhrase: string;
+  verdict: WordstatSourceGroupCriticVerdict;
+  reason: string;
+  resultCount: number;
+  exactCount: number;
+  signalCount: number;
+  driftSignalCount: number;
+  strictFitSignalCount: number;
+  sourceHasStrictFit: boolean;
+  sourceLooksWeak: boolean;
+  driftExamples: string[];
+  strictFitExamples: string[];
+};
 
 const HARD_TRASH_PATTERNS = [
   /(^|\s)(ваканси[ия]|работа|зарплата|резюме)(\s|$)/i,
@@ -249,6 +278,26 @@ const PRODUCT_TARIFF_QUALIFIER_PATTERN =
 const AI_KEYWORD_PATTERN = /(^|\s)(ai|ии)(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект|нейросет/i;
 const B2B_KEYWORD_FACET_PATTERN =
   /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|решен|платформ|стоимост|внедрен|купить|заказать|тариф|1с|crm|erp|bpm|workflow|digital|twin|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн|предприят|корпоратив/i;
+const SOURCE_GROUP_CRITIC_MIN_RESULTS = 4;
+const SOURCE_GROUP_CRITIC_MODEL_LIMIT = 24;
+const SOURCE_GROUP_CRITIC_DRIFT_PATTERNS = [
+  /(^|\s)(звук[а-яёa-z0-9-]*|слог[а-яёa-z0-9-]*|букв[а-яёa-z0-9-]*|реч[а-яёa-z0-9-]*|логопед[а-яёa-z0-9-]*|артикуляц[а-яёa-z0-9-]*|дошкольн[а-яёa-z0-9-]*|произношен[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(основн[а-яёa-z0-9-]*|базов[а-яёa-z0-9-]*)\s+(поняти[а-яёa-z0-9-]*|цель|цели|задач[а-яёa-z0-9-]*|принцип[а-яёa-z0-9-]*|вид[а-яёa-z0-9-]*|тип[а-яёa-z0-9-]*|функци[а-яёa-z0-9-]*|этап[а-яёa-z0-9-]*|направлен[а-яёa-z0-9-]*|уровн[а-яёa-z0-9-]*|средств[а-яёa-z0-9-]*)/i,
+  /(^|\s)(поняти[ея]|определен[а-яёa-z0-9-]*|цель|цели|задач[аи]?|принцип[а-яёa-z0-9-]*|виды|типы|функци[а-яёa-z0-9-]*|этапы|направлен[а-яёa-z0-9-]*|уровни|схем[а-яёa-z0-9-]*|структур[а-яёa-z0-9-]*|классификац[а-яёa-z0-9-]*|реферат|конспект|презентац[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(упражнен[а-яёa-z0-9-]*|задани[а-яёa-z0-9-]*|игр[а-яёa-z0-9-]*|картинк[а-яёa-z0-9-]*|стих[а-яёa-z0-9-]*)(\s|$)/i
+];
+const SOURCE_GROUP_CRITIC_WEAK_SOURCE_PATTERNS = [
+  /(^|\s)(предложени[еяюях]*|основн[а-яёa-z0-9-]*|поняти[а-яёa-z0-9-]*|цель|цели|задач[а-яёa-z0-9-]*|принцип[а-яёa-z0-9-]*|вид[а-яёa-z0-9-]*|тип[а-яёa-z0-9-]*|функци[а-яёa-z0-9-]*|этап[а-яёa-z0-9-]*|направлен[а-яёa-z0-9-]*)(\s|$)/i
+];
+const SOURCE_GROUP_STRICT_PRODUCT_FIT_PATTERNS = [
+  /(^|\s)(агент[а-яёa-z0-9-]*|ассистент[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(ai|ии|llm|ml|нейросет[а-яёa-z0-9-]*|искусственн[а-яёa-z0-9-]*\s+интеллект)\s+(агент[а-яёa-z0-9-]*|ассистент[а-яёa-z0-9-]*|платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*|сервис[а-яёa-z0-9-]*|для\s+(бизнес[а-яёa-z0-9-]*|компан[а-яёa-z0-9-]*|предприят[а-яёa-z0-9-]*|корпоративн[а-яёa-z0-9-]*))/i,
+  /(^|\s)(платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*|сервис[а-яёa-z0-9-]*)\s+(ai|ии|llm|ml|нейросет[а-яёa-z0-9-]*|искусственн[а-яёa-z0-9-]*\s+интеллект)(\s|$)/i,
+  /(^|\s)(b2b|enterprise|корпоративн[а-яёa-z0-9-]*|предприят[а-яёa-z0-9-]*|компан[а-яёa-z0-9-]*|бизнес[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(crm|erp|bpm|workflow|api|1с|1c|эдо|документооборот|закуп[а-яёa-z0-9-]*|тендер[а-яёa-z0-9-]*|договор[а-яёa-z0-9-]*|юридическ[а-яёa-z0-9-]*|финанс[а-яёa-z0-9-]*|поручен[а-яёa-z0-9-]*|согласован[а-яёa-z0-9-]*|база\s+знан[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(интеграц[а-яёa-z0-9-]*|внедрен[а-яёa-z0-9-]*|разработк[а-яёa-z0-9-]*|под\s+ключ|заказать|купить|стоимост[а-яёa-z0-9-]*|цен[а-яёa-z0-9-]*|платформ[а-яёa-z0-9-]*|решен[а-яёa-z0-9-]*)(\s|$)/i,
+  /(^|\s)(строител[а-яёa-z0-9-]*|производств[а-яёa-z0-9-]*|промышленн[а-яёa-z0-9-]*|беспилот[а-яёa-z0-9-]*|iot|телеметр[а-яёa-z0-9-]*|digital\s+twin|цифров[а-яёa-z0-9-]*\s+двойн[а-яёa-z0-9-]*|bim|бим)(\s|$)/i
+];
 const KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT = 60;
 const KEYWORD_CLEANING_MODEL_LANDING_BRIEF_LIMIT = 18;
 const KEYWORD_CLEANING_MODEL_NORMALIZATION_GROUP_LIMIT = 24;
@@ -712,6 +761,233 @@ function promoteExactDemandItems(market: MarketEnrichmentContract, items: Keywor
   });
 }
 
+function hasProtectedProjectFacet(value: string, guard: KeywordCleaningFacetGuard) {
+  const phraseStems = getKeywordCleaningFacetStems(value);
+
+  return guard.dominantProtectedStems.some((facet) => phraseStems.has(facet.stem));
+}
+
+function hasStrictRelatedProductFit(market: MarketEnrichmentContract, value: string, guard: KeywordCleaningFacetGuard) {
+  const normalizedValue = normalizePhrase(value);
+
+  if (hasProtectedProjectFacet(normalizedValue, guard)) {
+    return true;
+  }
+
+  if (hasPattern(normalizedValue, SOURCE_GROUP_STRICT_PRODUCT_FIT_PATTERNS)) {
+    return true;
+  }
+
+  return (
+    AI_KEYWORD_PATTERN.test(normalizedValue) &&
+    hasB2BKeywordFacet(normalizedValue) &&
+    !isConsumerAiNoisePhrase(normalizedValue) &&
+    !hasEducationIntentMismatch(market, normalizedValue)
+  );
+}
+
+function hasSourceGroupRelatedDriftSignal(market: MarketEnrichmentContract, phrase: string) {
+  const normalizedPhrase = normalizePhrase(phrase);
+
+  return (
+    hasPattern(normalizedPhrase, SOURCE_GROUP_CRITIC_DRIFT_PATTERNS) ||
+    isConsumerAiNoisePhrase(normalizedPhrase) ||
+    isOffContextRelatedNoisePhrase(normalizedPhrase) ||
+    hasThirdPartyBrandDemandMismatch(normalizedPhrase) ||
+    hasNonProductMarketIntentMismatch(normalizedPhrase) ||
+    hasEducationIntentMismatch(market, normalizedPhrase)
+  );
+}
+
+function sourceGroupLooksWeak(sourcePhrase: string, guard: KeywordCleaningFacetGuard) {
+  const normalizedSourcePhrase = normalizePhrase(sourcePhrase);
+
+  return hasPattern(normalizedSourcePhrase, SOURCE_GROUP_CRITIC_WEAK_SOURCE_PATTERNS) && !hasProtectedProjectFacet(normalizedSourcePhrase, guard);
+}
+
+function getSourceGroupCriticVerdict(input: {
+  driftShare: number;
+  driftSignalCount: number;
+  signalCount: number;
+  sourceHasStrictFit: boolean;
+  sourceLooksWeak: boolean;
+  strictFitShare: number;
+  strictFitSignalCount: number;
+}): WordstatSourceGroupCriticVerdict {
+  if (input.signalCount < SOURCE_GROUP_CRITIC_MIN_RESULTS) {
+    return "clean";
+  }
+
+  if (
+    input.sourceLooksWeak &&
+    input.driftSignalCount >= 2 &&
+    input.strictFitSignalCount <= 1 &&
+    input.strictFitShare < 0.5
+  ) {
+    return "source_drift";
+  }
+
+  if (!input.sourceHasStrictFit && input.driftShare >= 0.45 && input.strictFitShare < 0.45 && input.signalCount >= 6) {
+    return "source_drift";
+  }
+
+  if (input.driftShare >= 0.7 && input.strictFitShare < 0.35 && input.signalCount >= 6) {
+    return "source_drift";
+  }
+
+  if (input.driftShare >= 0.3 && input.strictFitShare < 0.6) {
+    return "mixed";
+  }
+
+  return "clean";
+}
+
+function buildWordstatSourceGroupCriticReason(critic: Omit<WordstatSourceGroupCritic, "reason">) {
+  if (critic.verdict === "source_drift") {
+    return `Source-group critic пометил Wordstat related-группу "${critic.sourcePhrase}" как source drift: ${critic.driftSignalCount}/${critic.signalCount} строк ушли в омонимический или справочный спрос, строгий product-fit держат ${critic.strictFitSignalCount}.`;
+  }
+
+  if (critic.verdict === "mixed") {
+    return `Source-group critic пометил Wordstat related-группу "${critic.sourcePhrase}" как mixed: в пачке есть drift-сигналы, поэтому related без строгого product-fit нельзя повышать в карту только по частотности.`;
+  }
+
+  return `Source-group critic не нашел групповой drift у Wordstat related-группы "${critic.sourcePhrase}".`;
+}
+
+function buildWordstatSourceGroupCritics(
+  market: MarketEnrichmentContract,
+  items: KeywordCleaningItem[]
+): Map<string, WordstatSourceGroupCritic> {
+  const guard = buildKeywordCleaningFacetGuard(market);
+  const groups = new Map<string, { sourcePhrase: string; items: KeywordCleaningItem[] }>();
+
+  for (const item of items) {
+    if (item.source !== "wordstat_related" || !item.sourcePhrase) {
+      continue;
+    }
+
+    const normalizedSourcePhrase = normalizePhrase(item.sourcePhrase);
+    const group = groups.get(normalizedSourcePhrase) ?? { items: [], sourcePhrase: item.sourcePhrase };
+    group.items.push(item);
+    groups.set(normalizedSourcePhrase, group);
+  }
+
+  const critics = new Map<string, WordstatSourceGroupCritic>();
+
+  for (const [normalizedSourcePhrase, group] of groups.entries()) {
+    const exactItems = group.items.filter((item) => item.frequency !== null && Boolean(item.wordstatResultId));
+    const signalItems = exactItems.length >= SOURCE_GROUP_CRITIC_MIN_RESULTS ? exactItems : group.items;
+    const driftItems = signalItems.filter((item) => hasSourceGroupRelatedDriftSignal(market, item.phrase));
+    const strictFitItems = signalItems.filter(
+      (item) => hasStrictRelatedProductFit(market, item.phrase, guard) && !hasSourceGroupRelatedDriftSignal(market, item.phrase)
+    );
+    const signalCount = signalItems.length;
+    const driftShare = signalCount > 0 ? driftItems.length / signalCount : 0;
+    const strictFitShare = signalCount > 0 ? strictFitItems.length / signalCount : 0;
+    const sourceHasStrictFit = hasStrictRelatedProductFit(market, group.sourcePhrase, guard);
+    const weakSource = sourceGroupLooksWeak(group.sourcePhrase, guard);
+    const baseCritic = {
+      driftExamples: driftItems.slice(0, 5).map((item) => item.phrase),
+      driftSignalCount: driftItems.length,
+      exactCount: exactItems.length,
+      normalizedSourcePhrase,
+      resultCount: group.items.length,
+      signalCount,
+      sourceHasStrictFit,
+      sourceLooksWeak: weakSource,
+      sourcePhrase: group.sourcePhrase,
+      strictFitExamples: strictFitItems.slice(0, 5).map((item) => item.phrase),
+      strictFitSignalCount: strictFitItems.length,
+      verdict: getSourceGroupCriticVerdict({
+        driftShare,
+        driftSignalCount: driftItems.length,
+        signalCount,
+        sourceHasStrictFit,
+        sourceLooksWeak: weakSource,
+        strictFitShare,
+        strictFitSignalCount: strictFitItems.length
+      })
+    };
+    const critic: WordstatSourceGroupCritic = {
+      ...baseCritic,
+      reason: buildWordstatSourceGroupCriticReason(baseCritic)
+    };
+
+    critics.set(normalizedSourcePhrase, critic);
+  }
+
+  return critics;
+}
+
+function getSourceGroupCriticEvidenceRef(critic: WordstatSourceGroupCritic) {
+  return `backend:source-group-critic:${slugify(critic.sourcePhrase)}`;
+}
+
+function enforceWordstatSourceGroupCritic(
+  market: MarketEnrichmentContract,
+  items: KeywordCleaningItem[]
+): KeywordCleaningItem[] {
+  const guard = buildKeywordCleaningFacetGuard(market);
+  const critics = buildWordstatSourceGroupCritics(market, items);
+
+  return items.map((item) => {
+    if (item.source !== "wordstat_related" || !item.sourcePhrase) {
+      return item;
+    }
+
+    const critic = critics.get(normalizePhrase(item.sourcePhrase));
+
+    if (!critic || critic.verdict === "clean") {
+      return item;
+    }
+
+    const selfHasDriftSignal = hasSourceGroupRelatedDriftSignal(market, item.phrase);
+    const selfHasStrictFit = hasStrictRelatedProductFit(market, item.phrase, guard) && !selfHasDriftSignal;
+    const blocker = critic.verdict === "source_drift" ? "source_group_related_drift" : "source_group_related_mixed";
+    const evidenceRefs = unique([...item.evidenceRefs, getSourceGroupCriticEvidenceRef(critic)]);
+    const blockers = unique([...(Array.isArray(item.blockers) ? item.blockers : []), blocker]);
+
+    if (critic.verdict === "mixed") {
+      if ((item.decision !== "use" && item.decision !== "support") || selfHasStrictFit) {
+        return {
+          ...item,
+          blockers,
+          evidenceRefs
+        };
+      }
+
+      return {
+        ...item,
+        blockers,
+        confidence: Math.min(item.confidence, 0.58),
+        decision: "risky",
+        evidenceRefs,
+        reason: `${critic.reason} Backend guard понизил related без строгого product-fit в risky: mixed-группа требует ручной проверки.`
+      };
+    }
+
+    if (selfHasStrictFit) {
+      return {
+        ...item,
+        blockers,
+        confidence: Math.min(item.confidence, 0.58),
+        decision: "risky",
+        evidenceRefs,
+        reason: `${critic.reason} Фраза сама держит product-fit, поэтому не удаляется, но не может попасть в approved lanes без ручной проверки source-группы.`
+      };
+    }
+
+    return {
+      ...item,
+      blockers,
+      confidence: Math.max(item.confidence, 0.9),
+      decision: "trash",
+      evidenceRefs,
+      reason: `${critic.reason} Backend guard отбросил фразу до keyword map: загрязненная source-группа не должна протаскивать омонимический related-спрос.`
+    };
+  });
+}
+
 function enforceKeywordCleaningNoiseGuard(items: KeywordCleaningItem[]): KeywordCleaningItem[] {
   return items.map((item) => {
     if (isOffContextRelatedNoisePhrase(item.normalizedPhrase)) {
@@ -756,7 +1032,10 @@ function enforceKeywordCleaningBackendGuards(
   items: KeywordCleaningItem[]
 ): KeywordCleaningItem[] {
   return enforceKeywordCleaningNoiseGuard(
-    promoteExactDemandItems(market, rescueExactTrashItems(market, enforceKeywordCleaningSemanticGuard(market, items)))
+    enforceWordstatSourceGroupCritic(
+      market,
+      promoteExactDemandItems(market, rescueExactTrashItems(market, enforceKeywordCleaningSemanticGuard(market, items)))
+    )
   );
 }
 
@@ -1116,6 +1395,15 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key
     .sort((left, right) => right.score - left.score || left.index - right.index)
     .slice(0, KEYWORD_CLEANING_MODEL_TOP_RESULT_LIMIT)
     .map((item) => item.result);
+  const modelSourceGroupCritics = [...buildWordstatSourceGroupCritics(contract, getBaseItems(contract)).values()]
+    .filter((critic) => critic.verdict !== "clean")
+    .sort(
+      (left, right) =>
+        (right.verdict === "source_drift" ? 1 : 0) - (left.verdict === "source_drift" ? 1 : 0) ||
+        right.driftSignalCount - left.driftSignalCount ||
+        right.resultCount - left.resultCount
+    )
+    .slice(0, SOURCE_GROUP_CRITIC_MODEL_LIMIT);
 
   return {
     approvedAnchors: contract.anchorReview.wordstatQueue.slice(0, KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT).map((anchor) => ({
@@ -1170,6 +1458,17 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key
       sourceClusterId: result.sourceClusterId,
       sourceClusterTitle: truncateTaskText(result.sourceClusterTitle),
       sourcePhrase: result.sourcePhrase
+    })),
+    sourceGroupCritics: modelSourceGroupCritics.map((critic) => ({
+      driftExamples: critic.driftExamples.map((phrase) => truncateTaskText(phrase)),
+      driftSignalCount: critic.driftSignalCount,
+      exactCount: critic.exactCount,
+      reason: truncateTaskText(critic.reason, 220),
+      resultCount: critic.resultCount,
+      sourcePhrase: truncateTaskText(critic.sourcePhrase),
+      strictFitExamples: critic.strictFitExamples.map((phrase) => truncateTaskText(phrase)),
+      strictFitSignalCount: critic.strictFitSignalCount,
+      verdict: critic.verdict
     }))
   };
 }
@@ -1439,6 +1738,8 @@ function buildKeywordCleaningModelTask(
       broadFrequencyDoesNotOverrideSemanticFit: true,
       demoteRelatedWhenDominantFacetMissing: false,
       preserveApprovedAnchorFacets: true,
+      quarantineNoisyRelatedGroups: true,
+      sourceGroupDriftDoesNotEnterKeywordMap: true,
       riskyWhenNoPageBinding: false,
       trashWhenOffContext: true,
       useRequiresExactEvidence: true
@@ -1494,6 +1795,7 @@ function buildKeywordCleaningModelTask(
       "Отбрасывать бытовой AI/нейросетевой шум: онлайн, бесплатно, тексты, песни, фото, видео, чат, порно и похожие consumer-запросы не являются B2B SEO-кандидатами без source evidence.",
       "Образовательный спрос (курсы, обучение, тренинги, сертификаты) не отправлять в product keyword map, если Product Understanding/normalization не подтверждает образовательный offer сайта.",
       "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря; глобальный доминантный термин проекта не должен запрещать отдельные подтверждённые ветки спроса.",
+      "Проверять sourceGroupCritics после Wordstat: source_drift-группы не отправлять в keyword map, mixed-группы повышать только при строгом product/source fit у конкретной фразы.",
       "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.",
       "Не сохранять rewrite/apply decisions."
     ],

seo_mode/seo_mode/server/src/scripts/checkDemandCoverage.ts

@@ -160,6 +160,15 @@ assert.ok(
     keywordCleaningSource.includes("getKeywordCleaningModelTopResultScore"),
   "Keyword cleaning model task must keep a slim ranked payload so AI Workspace does not fail with request entity too large."
 );
+assert.ok(
+  keywordCleaningSource.includes("buildWordstatSourceGroupCritics") &&
+    keywordCleaningSource.includes("enforceWordstatSourceGroupCritic") &&
+    keywordCleaningSource.includes("source_group_related_drift") &&
+    keywordCleaningSource.includes("sourceGroupCritics") &&
+    keywordCleaningSource.includes("quarantineNoisyRelatedGroups") &&
+    keywordCleaningSource.includes("sourceGroupDriftDoesNotEnterKeywordMap"),
+  "Keyword cleaning must run a source-group/related-fit critic after Wordstat so homonym related batches are quarantined before keyword map."
+);
 assert.ok(
   modelProviderRegistrySource.includes('"seo.market_frontier_discovery"'),
   "Model provider registry must expose seo.market_frontier_discovery as a runnable task type."