fix(seo): guard keyword map semantic fit

seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts

@@ -89,11 +89,28 @@ export type KeywordCleaningModelTaskContract = {
     decisionValues: KeywordCleaningDecision[];
   };
   decisionPolicy: {
+    broadFrequencyDoesNotOverrideSemanticFit: boolean;
+    demoteRelatedWhenDominantFacetMissing: boolean;
+    preserveApprovedAnchorFacets: boolean;
     useRequiresExactEvidence: boolean;
     riskyWhenNoPageBinding: boolean;
     trashWhenOffContext: boolean;
     articleDoesNotEnterRewrite: boolean;
   };
+  semanticGuard: {
+    approvedAnchorCount: number;
+    dominantProtectedFacets: Array<{
+      approvedAnchorCount: number;
+      approvedAnchorShare: number;
+      examples: string[];
+      stem: string;
+    }>;
+    policy: {
+      broadFrequencyDoesNotOverrideSemanticFit: boolean;
+      demoteRelatedWhenDominantFacetMissing: boolean;
+      preserveSourceAnchorFacetsForRelated: boolean;
+    };
+  };
   stopConditions: string[];
 };
 
@@ -208,6 +225,110 @@ function wordCount(value: string) {
   return normalizePhrase(value).split(/\s+/).filter(Boolean).length;
 }
 
+const KEYWORD_CLEANING_FACET_STOP_WORDS = new Set([
+  "a",
+  "an",
+  "and",
+  "by",
+  "for",
+  "in",
+  "of",
+  "on",
+  "or",
+  "the",
+  "to",
+  "без",
+  "в",
+  "для",
+  "и",
+  "или",
+  "как",
+  "на",
+  "о",
+  "об",
+  "от",
+  "по",
+  "при",
+  "с",
+  "со",
+  "что",
+  "это"
+]);
+
+const GENERIC_KEYWORD_CLEANING_FACET_STEMS = new Set([
+  "автоматизац",
+  "автоматизаци",
+  "бизнес",
+  "данн",
+  "задач",
+  "инструмент",
+  "компан",
+  "контур",
+  "модул",
+  "платформ",
+  "приложен",
+  "процесс",
+  "проект",
+  "разработк",
+  "решен",
+  "сервис",
+  "систем",
+  "сайт",
+  "управлен",
+  "услуг"
+]);
+
+function stemKeywordCleaningToken(token: string) {
+  const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е");
+
+  if (/^(ai|ии)$/i.test(normalizedToken) || /^(artificial|intelligence)$/i.test(normalizedToken)) {
+    return "ai";
+  }
+
+  if (/^(искусствен|интеллект)/i.test(normalizedToken)) {
+    return "ai";
+  }
+
+  if (/^[a-z0-9]+$/i.test(normalizedToken)) {
+    return normalizedToken.toLowerCase();
+  }
+
+  return normalizedToken.replace(
+    /(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i,
+    ""
+  );
+}
+
+function getKeywordCleaningFacetStems(value: string) {
+  return new Set(
+    normalizePhrase(value)
+      .replace(/ё/g, "е")
+      .split(/[^a-zа-я0-9]+/i)
+      .map((token) => token.trim())
+      .filter((token) => token.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(token))
+      .map(stemKeywordCleaningToken)
+      .filter((stem) => stem.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(stem))
+  );
+}
+
+function isProtectedKeywordCleaningFacetStem(
+  stem: string,
+  approvedAnchorCount: number,
+  stemDocumentCounts: Map<string, number>
+) {
+  if (GENERIC_KEYWORD_CLEANING_FACET_STEMS.has(stem)) {
+    return false;
+  }
+
+  if (/^[a-z0-9]+$/i.test(stem)) {
+    return true;
+  }
+
+  const documentShare = approvedAnchorCount > 0 ? (stemDocumentCounts.get(stem) ?? 0) / approvedAnchorCount : 0;
+
+  return stem.length >= 7 || documentShare >= 0.18;
+}
+
 function hasPattern(value: string, patterns: RegExp[]) {
   return patterns.some((pattern) => pattern.test(value));
 }
@@ -300,6 +421,134 @@ function buildBriefClusterMap(contract: MarketEnrichmentContract) {
   return clusterMap;
 }
 
+type KeywordCleaningFacetGuard = {
+  approvedAnchorCount: number;
+  dominantProtectedStems: Array<{ count: number; stem: string }>;
+  examplesByStem: Map<string, string[]>;
+  stemDocumentCounts: Map<string, number>;
+};
+
+function buildKeywordCleaningFacetGuard(contract: MarketEnrichmentContract): KeywordCleaningFacetGuard {
+  const stemDocumentCounts = new Map<string, number>();
+  const examplesByStem = new Map<string, string[]>();
+  const approvedAnchors = contract.anchorReview.wordstatQueue.slice(0, 120);
+
+  for (const anchor of approvedAnchors) {
+    for (const stem of getKeywordCleaningFacetStems(`${anchor.phrase} ${anchor.clusterTitle}`)) {
+      stemDocumentCounts.set(stem, (stemDocumentCounts.get(stem) ?? 0) + 1);
+
+      const examples = examplesByStem.get(stem) ?? [];
+
+      if (examples.length < 3 && !examples.map(normalizePhrase).includes(normalizePhrase(anchor.phrase))) {
+        examples.push(anchor.phrase);
+        examplesByStem.set(stem, examples);
+      }
+    }
+  }
+
+  const dominantProtectedStems = [...stemDocumentCounts.entries()]
+    .filter(([stem, count]) => {
+      const share = approvedAnchors.length > 0 ? count / approvedAnchors.length : 0;
+
+      return share >= 0.3 && isProtectedKeywordCleaningFacetStem(stem, approvedAnchors.length, stemDocumentCounts);
+    })
+    .sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0], "ru"))
+    .slice(0, 12)
+    .map(([stem, count]) => ({ count, stem }));
+
+  return {
+    approvedAnchorCount: approvedAnchors.length,
+    dominantProtectedStems,
+    examplesByStem,
+    stemDocumentCounts
+  };
+}
+
+function buildKeywordCleaningSemanticGuard(contract: MarketEnrichmentContract): KeywordCleaningModelTaskContract["semanticGuard"] {
+  const guard = buildKeywordCleaningFacetGuard(contract);
+  const dominantProtectedFacets = guard.dominantProtectedStems
+    .map((facet) => ({
+      approvedAnchorCount: facet.count,
+      approvedAnchorShare: Number((facet.count / Math.max(guard.approvedAnchorCount, 1)).toFixed(3)),
+      examples: guard.examplesByStem.get(facet.stem) ?? [],
+      stem: facet.stem
+    }));
+
+  return {
+    approvedAnchorCount: guard.approvedAnchorCount,
+    dominantProtectedFacets,
+    policy: {
+      broadFrequencyDoesNotOverrideSemanticFit: true,
+      demoteRelatedWhenDominantFacetMissing: true,
+      preserveSourceAnchorFacetsForRelated: true
+    }
+  };
+}
+
+function getProtectedSourceFacetStems(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) {
+  if (item.source !== "wordstat_related" || !item.sourcePhrase) {
+    return new Set<string>();
+  }
+
+  const sourceStems = getKeywordCleaningFacetStems(`${item.sourcePhrase} ${item.clusterTitle}`);
+
+  return new Set(
+    [...sourceStems].filter((stem) =>
+      isProtectedKeywordCleaningFacetStem(stem, guard.approvedAnchorCount, guard.stemDocumentCounts)
+    )
+  );
+}
+
+function getKeywordCleaningSemanticDowngradeReason(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) {
+  const phraseStems = getKeywordCleaningFacetStems(item.phrase);
+  const sourceProtectedStems = getProtectedSourceFacetStems(item, guard);
+  const missingSourceStems = [...sourceProtectedStems].filter((stem) => !phraseStems.has(stem));
+
+  if (sourceProtectedStems.size > 0 && missingSourceStems.length === sourceProtectedStems.size) {
+    return `Wordstat related потерял protected-признак исходного якоря (${missingSourceStems.join(", ")}).`;
+  }
+
+  const dominantProtectedStems = guard.dominantProtectedStems.map((item) => item.stem);
+  const missingDominantStems = dominantProtectedStems.filter((stem) => !phraseStems.has(stem));
+
+  if (dominantProtectedStems.length > 0 && missingDominantStems.length === dominantProtectedStems.length) {
+    return `Фраза не содержит доминантный protected-смысл approved anchors (${missingDominantStems.join(", ")}).`;
+  }
+
+  return null;
+}
+
+function enforceKeywordCleaningSemanticGuard(
+  market: MarketEnrichmentContract,
+  items: KeywordCleaningItem[]
+): KeywordCleaningItem[] {
+  const guard = buildKeywordCleaningFacetGuard(market);
+
+  if (guard.approvedAnchorCount === 0) {
+    return items;
+  }
+
+  return items.map((item) => {
+    if (item.decision !== "use" && item.decision !== "support") {
+      return item;
+    }
+
+    const semanticDowngradeReason = getKeywordCleaningSemanticDowngradeReason(item, guard);
+
+    if (!semanticDowngradeReason) {
+      return item;
+    }
+
+    return {
+      ...item,
+      blockers: unique([...item.blockers, "semantic_facet_loss"]),
+      confidence: Math.min(item.confidence, 0.62),
+      decision: "risky",
+      reason: `${semanticDowngradeReason} Backend guard понизил фразу в risky: частотность не перекрывает потерю смысла.`
+    };
+  });
+}
+
 function getTargetPath(seed: SeedQueueItem, briefPhraseMap: Map<string, string>, briefClusterMap: Map<string, string>) {
   const directTarget = briefPhraseMap.get(normalizePhrase(seed.phrase));
 
@@ -704,7 +953,11 @@ function buildReadiness(items: KeywordCleaningItem[], lanes: KeywordCleaningCont
     articleCount: lanes.article.length,
     exactEvidenceCount: items.filter((item) => item.frequency !== null).length,
     keywordMapCandidateCount: items.filter(
-      (item) => item.decision !== "trash" && item.evidenceStatus === "collected" && item.frequency !== null && item.targetPath
+      (item) =>
+        (item.decision === "use" || item.decision === "support") &&
+        item.evidenceStatus === "collected" &&
+        item.frequency !== null &&
+        item.targetPath
     ).length,
     missingPageBindingCount: items.filter((item) => item.targetPath === null && item.decision !== "trash").length,
     riskyCount: lanes.risky.length,
@@ -734,6 +987,9 @@ function buildKeywordCleaningModelTask(
     ],
     decisionPolicy: {
       articleDoesNotEnterRewrite: true,
+      broadFrequencyDoesNotOverrideSemanticFit: true,
+      demoteRelatedWhenDominantFacetMissing: true,
+      preserveApprovedAnchorFacets: true,
       riskyWhenNoPageBinding: true,
       trashWhenOffContext: true,
       useRequiresExactEvidence: true
@@ -777,9 +1033,11 @@ function buildKeywordCleaningModelTask(
     providerMode: "model_provider_contract",
     schemaVersion: "seo-keyword-cleaning-task.v1",
     semanticRunId: contract.semanticRunId,
+    semanticGuard: buildKeywordCleaningSemanticGuard(contract),
     stopConditions: [
       "Не отправлять trash в keyword map. Risky/article exact-фразы можно только предсортировать в Stage 3, но не считать approved без пользователя.",
       "Не придумывать спрос без Wordstat/SERP evidence.",
+      "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря или доминантный смысл approved anchors.",
       "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.",
       "Не сохранять rewrite/apply decisions."
     ],
@@ -824,7 +1082,7 @@ function buildNextActions(contract: Omit<KeywordCleaningContract, "nextActions">
 
 function buildFallbackContract(projectId: string, market: MarketEnrichmentContract): KeywordCleaningContract {
   const modelTask = buildKeywordCleaningModelTask(projectId, market);
-  const items = classifyItems(market, getBaseItems(market));
+  const items = enforceKeywordCleaningSemanticGuard(market, classifyItems(market, getBaseItems(market)));
   const lanes = buildLanes(items);
   const readiness = buildReadiness(items, lanes);
   const contractWithoutActions: Omit<KeywordCleaningContract, "nextActions"> = {
@@ -990,13 +1248,34 @@ export async function getKeywordCleaningContract(projectId: string): Promise<Key
 
   const latest = await getLatestAlignedKeywordCleaning(projectId, market.semanticRunId, market.wordstat.latestJob?.id ?? null);
 
-  return latest
+  if (!latest) {
-    ? {
+    return fallbackContract;
-        ...latest,
-        modelTask: fallbackContract.modelTask,
-        sourceTaskId: fallbackContract.sourceTaskId
   }
-    : fallbackContract;
+
+  const items = enforceKeywordCleaningSemanticGuard(market, latest.items);
+  const lanes = buildLanes(items);
+  const readiness = buildReadiness(items, lanes);
+
+  return {
+    ...latest,
+    items,
+    lanes,
+    modelTask: fallbackContract.modelTask,
+    nextActions: buildNextActions({
+      ...latest,
+      items,
+      lanes,
+      modelTask: fallbackContract.modelTask,
+      readiness,
+      sourceTaskId: fallbackContract.sourceTaskId
+    }),
+    readiness,
+    sourceTaskId: fallbackContract.sourceTaskId,
+    summary:
+      items.length > 0
+        ? `Очищено ${items.length} фраз: ${readiness.useCount} use, ${readiness.supportCount} support, ${readiness.articleCount} article, ${readiness.riskyCount} risky, ${readiness.trashCount} trash.`
+        : "Нет фраз для keyword cleaning."
+  };
 }
 
 export async function saveKeywordCleaningManual(
@@ -1051,7 +1330,7 @@ export async function saveKeywordCleaningFromModelProvider(
 
   const alignedOutput: KeywordCleaningContract = {
     ...output,
-    items: mergeModelProviderItemsWithMarketEvidence(output.items, market),
+    items: enforceKeywordCleaningSemanticGuard(market, mergeModelProviderItemsWithMarketEvidence(output.items, market)),
     modelTask,
     projectOntologyVersion: market.projectOntologyVersion,
     semanticRunId: market.semanticRunId,

seo_mode/seo_mode/server/src/keywords/keywordMap.ts

@@ -103,30 +103,6 @@ function getKeywordProfileText(item: Pick<KeywordCleaningItem, "clusterTitle" |
   return normalizePhrase(`${item.phrase} ${item.clusterTitle}`);
 }
 
-function isAiDifferentiatorCandidate(item: Pick<KeywordCleaningItem, "clusterTitle" | "phrase">) {
-  const text = getKeywordProfileText(item);
-
-  return /(^|\s)(ai|ии)(\s|$)/i.test(text) || /(агент|агентн|ассистент|assistant|agentic)/i.test(text);
-}
-
-function hasProductLikeHybridSignal(value: string) {
-  const text = normalizePhrase(value);
-  const hasProductNoun = /(систем[ауы]?|платформ[ауы]?|сервис|software|product|suite)/i.test(text);
-  const hasProcessNoun = /(бизнес[-\s]?процесс|bpms?|workflow|процесс[а-яё]*)/i.test(text);
-
-  return hasProductNoun && hasProcessNoun;
-}
-
-function isBroadArticleBacklogPhrase(value: string) {
-  const text = normalizePhrase(value);
-
-  return /^автоматизаци[яи]\s+бизнес[-\s]?процесс(ов|ы)?$/i.test(text);
-}
-
-function isHybridSecondaryCandidate(item: KeywordCleaningItem) {
-  return item.frequency !== null && hasProductLikeHybridSignal(item.phrase) && !isAiDifferentiatorCandidate(item);
-}
-
 function isReviewedOntology(version: MarketEnrichmentContract["projectOntologyVersion"]) {
   return (
     Boolean(version) &&
@@ -214,6 +190,17 @@ type SemanticFacetGuard = {
   stemDocumentCounts: Map<string, number>;
 };
 
+type SemanticFitAssessment = {
+  label: "lost_project_facet" | "lost_source_facet" | "strong" | "weak";
+  matchedProjectStems: string[];
+  matchedSourceStems: string[];
+  missingProjectStems: string[];
+  missingSourceStems: string[];
+  projectCoverage: number | null;
+  score: number;
+  sourceCoverage: number | null;
+};
+
 function stemKeywordToken(token: string) {
   const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е");
 
@@ -318,14 +305,62 @@ function hasSemanticFacetLoss(item: KeywordCleaningItem, guard: SemanticFacetGua
   return getMissingSemanticFacetStems(item, guard).length > 0;
 }
 
-function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) {
+function getCoverage(matchedCount: number, totalCount: number) {
-  if (guard.dominantProtectedStems.size === 0) {
+  return totalCount > 0 ? matchedCount / totalCount : null;
-    return false;
 }
 
+function getSemanticFitAssessment(item: KeywordCleaningItem, guard: SemanticFacetGuard): SemanticFitAssessment {
   const phraseStems = getKeywordFacetStems(item.phrase);
+  const sourceStems = item.source === "wordstat_related" ? getProtectedSourceFacetStems(item, guard) : new Set<string>();
+  const projectStems = guard.dominantProtectedStems;
+  const matchedSourceStems = [...sourceStems].filter((stem) => phraseStems.has(stem));
+  const matchedProjectStems = [...projectStems].filter((stem) => phraseStems.has(stem));
+  const missingSourceStems = [...sourceStems].filter((stem) => !phraseStems.has(stem));
+  const missingProjectStems = [...projectStems].filter((stem) => !phraseStems.has(stem));
+  const sourceCoverage = getCoverage(matchedSourceStems.length, sourceStems.size);
+  const projectCoverage = getCoverage(matchedProjectStems.length, projectStems.size);
+  const score = Math.min(sourceCoverage ?? 1, projectCoverage ?? 1);
 
-  return [...guard.dominantProtectedStems].every((stem) => !phraseStems.has(stem));
+  if (sourceCoverage === 0 && sourceStems.size > 0) {
+    return {
+      label: "lost_source_facet",
+      matchedProjectStems,
+      matchedSourceStems,
+      missingProjectStems,
+      missingSourceStems,
+      projectCoverage,
+      score,
+      sourceCoverage
+    };
+  }
+
+  if (projectCoverage === 0 && projectStems.size > 0) {
+    return {
+      label: "lost_project_facet",
+      matchedProjectStems,
+      matchedSourceStems,
+      missingProjectStems,
+      missingSourceStems,
+      projectCoverage,
+      score,
+      sourceCoverage
+    };
+  }
+
+  return {
+    label: score >= 0.5 ? "strong" : "weak",
+    matchedProjectStems,
+    matchedSourceStems,
+    missingProjectStems,
+    missingSourceStems,
+    projectCoverage,
+    score,
+    sourceCoverage
+  };
+}
+
+function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) {
+  return getSemanticFitAssessment(item, guard).label === "lost_project_facet";
 }
 
 function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) {
@@ -335,13 +370,16 @@ function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) {
 }
 
 function canModelRouteKeyword(item: KeywordCleaningItem, guard?: SemanticFacetGuard) {
+  const semanticFit = guard ? getSemanticFitAssessment(item, guard) : null;
+
   return (
+    (item.decision === "use" || item.decision === "support") &&
     item.evidenceStatus === "collected" &&
     item.frequency !== null &&
     Boolean(item.projectOntologyVersionId) &&
     Boolean(item.wordstatResultId) &&
     Boolean(item.targetPath) &&
-    (!guard || (!hasSemanticFacetLoss(item, guard) && !missesDominantProjectFacet(item, guard)))
+    (!semanticFit || (semanticFit.label === "strong" && semanticFit.score >= 0.5))
   );
 }
 
@@ -374,14 +412,14 @@ function getKeywordRole(item: KeywordCleaningItem, indexInTarget: number, guard:
     return "support";
   }
 
-  if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") {
-    return "secondary";
-  }
-
   if (isClarifyingKeywordCandidate(item)) {
     return "differentiator";
   }
 
+  if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") {
+    return "secondary";
+  }
+
   if (item.priority === "high" || item.priority === "medium") {
     return "secondary";
   }
@@ -439,6 +477,13 @@ function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleanin
         return rightPriorityWeight - leftPriorityWeight;
       }
 
+      const leftSemanticFit = getSemanticFitAssessment(left, semanticFacetGuard).score;
+      const rightSemanticFit = getSemanticFitAssessment(right, semanticFacetGuard).score;
+
+      if (leftSemanticFit !== rightSemanticFit) {
+        return rightSemanticFit - leftSemanticFit;
+      }
+
       const leftFrequency = left.frequency ?? -1;
       const rightFrequency = right.frequency ?? -1;
 
@@ -514,15 +559,15 @@ function getKeywordMapItemReason(
   }
 
   if (role === "validate" && hasSemanticFacetLoss(item, semanticFacetGuard)) {
-    return "Wordstat related потерял доминантный признак исходного якоря: оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта.";
+    const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard);
+
+    return `Wordstat related потерял protected-признак исходного якоря (${semanticFit.missingSourceStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта.`;
   }
 
   if (role === "validate" && missesDominantProjectFacet(item, semanticFacetGuard)) {
-    return "Фраза не содержит доминантный смысл из подтверждённой очереди якорей: оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически.";
+    const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard);
-  }
 
-  if (isBroadArticleBacklogPhrase(item.phrase) && role === "validate") {
+    return `Фраза не содержит доминантный protected-смысл из подтверждённой очереди (${semanticFit.missingProjectStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически.`;
-    return "Широкая head-фраза уходит в article/backlog: не делаем её прямой основной ставкой текущей посадочной без отдельного SERP page-type решения.";
   }
 
   if (item.decision === "article") {