fix(seo): guard keyword map semantic fit

This commit is contained in:
DCCONSTRUCTIONS 2026-07-03 09:05:01 +03:00
parent 1ed099140f
commit 43d2af454d
2 changed files with 373 additions and 49 deletions

View File

@ -89,11 +89,28 @@ export type KeywordCleaningModelTaskContract = {
decisionValues: KeywordCleaningDecision[];
};
decisionPolicy: {
broadFrequencyDoesNotOverrideSemanticFit: boolean;
demoteRelatedWhenDominantFacetMissing: boolean;
preserveApprovedAnchorFacets: boolean;
useRequiresExactEvidence: boolean;
riskyWhenNoPageBinding: boolean;
trashWhenOffContext: boolean;
articleDoesNotEnterRewrite: boolean;
};
semanticGuard: {
approvedAnchorCount: number;
dominantProtectedFacets: Array<{
approvedAnchorCount: number;
approvedAnchorShare: number;
examples: string[];
stem: string;
}>;
policy: {
broadFrequencyDoesNotOverrideSemanticFit: boolean;
demoteRelatedWhenDominantFacetMissing: boolean;
preserveSourceAnchorFacetsForRelated: boolean;
};
};
stopConditions: string[];
};
@ -208,6 +225,110 @@ function wordCount(value: string) {
return normalizePhrase(value).split(/\s+/).filter(Boolean).length;
}
const KEYWORD_CLEANING_FACET_STOP_WORDS = new Set([
"a",
"an",
"and",
"by",
"for",
"in",
"of",
"on",
"or",
"the",
"to",
"без",
"в",
"для",
"и",
"или",
"как",
"на",
"о",
"об",
"от",
"по",
"при",
"с",
"со",
"что",
"это"
]);
const GENERIC_KEYWORD_CLEANING_FACET_STEMS = new Set([
"автоматизац",
"автоматизаци",
"бизнес",
"данн",
"задач",
"инструмент",
"компан",
"контур",
"модул",
"платформ",
"приложен",
"процесс",
"проект",
"разработк",
"решен",
"сервис",
"систем",
"сайт",
"управлен",
"услуг"
]);
function stemKeywordCleaningToken(token: string) {
const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е");
if (/^(ai|ии)$/i.test(normalizedToken) || /^(artificial|intelligence)$/i.test(normalizedToken)) {
return "ai";
}
if (/^(искусствен|интеллект)/i.test(normalizedToken)) {
return "ai";
}
if (/^[a-z0-9]+$/i.test(normalizedToken)) {
return normalizedToken.toLowerCase();
}
return normalizedToken.replace(
/(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i,
""
);
}
function getKeywordCleaningFacetStems(value: string) {
return new Set(
normalizePhrase(value)
.replace(/ё/g, "е")
.split(/[^a-zа-я0-9]+/i)
.map((token) => token.trim())
.filter((token) => token.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(token))
.map(stemKeywordCleaningToken)
.filter((stem) => stem.length > 1 && !KEYWORD_CLEANING_FACET_STOP_WORDS.has(stem))
);
}
function isProtectedKeywordCleaningFacetStem(
stem: string,
approvedAnchorCount: number,
stemDocumentCounts: Map<string, number>
) {
if (GENERIC_KEYWORD_CLEANING_FACET_STEMS.has(stem)) {
return false;
}
if (/^[a-z0-9]+$/i.test(stem)) {
return true;
}
const documentShare = approvedAnchorCount > 0 ? (stemDocumentCounts.get(stem) ?? 0) / approvedAnchorCount : 0;
return stem.length >= 7 || documentShare >= 0.18;
}
function hasPattern(value: string, patterns: RegExp[]) {
return patterns.some((pattern) => pattern.test(value));
}
@ -300,6 +421,134 @@ function buildBriefClusterMap(contract: MarketEnrichmentContract) {
return clusterMap;
}
type KeywordCleaningFacetGuard = {
approvedAnchorCount: number;
dominantProtectedStems: Array<{ count: number; stem: string }>;
examplesByStem: Map<string, string[]>;
stemDocumentCounts: Map<string, number>;
};
function buildKeywordCleaningFacetGuard(contract: MarketEnrichmentContract): KeywordCleaningFacetGuard {
const stemDocumentCounts = new Map<string, number>();
const examplesByStem = new Map<string, string[]>();
const approvedAnchors = contract.anchorReview.wordstatQueue.slice(0, 120);
for (const anchor of approvedAnchors) {
for (const stem of getKeywordCleaningFacetStems(`${anchor.phrase} ${anchor.clusterTitle}`)) {
stemDocumentCounts.set(stem, (stemDocumentCounts.get(stem) ?? 0) + 1);
const examples = examplesByStem.get(stem) ?? [];
if (examples.length < 3 && !examples.map(normalizePhrase).includes(normalizePhrase(anchor.phrase))) {
examples.push(anchor.phrase);
examplesByStem.set(stem, examples);
}
}
}
const dominantProtectedStems = [...stemDocumentCounts.entries()]
.filter(([stem, count]) => {
const share = approvedAnchors.length > 0 ? count / approvedAnchors.length : 0;
return share >= 0.3 && isProtectedKeywordCleaningFacetStem(stem, approvedAnchors.length, stemDocumentCounts);
})
.sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0], "ru"))
.slice(0, 12)
.map(([stem, count]) => ({ count, stem }));
return {
approvedAnchorCount: approvedAnchors.length,
dominantProtectedStems,
examplesByStem,
stemDocumentCounts
};
}
function buildKeywordCleaningSemanticGuard(contract: MarketEnrichmentContract): KeywordCleaningModelTaskContract["semanticGuard"] {
const guard = buildKeywordCleaningFacetGuard(contract);
const dominantProtectedFacets = guard.dominantProtectedStems
.map((facet) => ({
approvedAnchorCount: facet.count,
approvedAnchorShare: Number((facet.count / Math.max(guard.approvedAnchorCount, 1)).toFixed(3)),
examples: guard.examplesByStem.get(facet.stem) ?? [],
stem: facet.stem
}));
return {
approvedAnchorCount: guard.approvedAnchorCount,
dominantProtectedFacets,
policy: {
broadFrequencyDoesNotOverrideSemanticFit: true,
demoteRelatedWhenDominantFacetMissing: true,
preserveSourceAnchorFacetsForRelated: true
}
};
}
function getProtectedSourceFacetStems(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) {
if (item.source !== "wordstat_related" || !item.sourcePhrase) {
return new Set<string>();
}
const sourceStems = getKeywordCleaningFacetStems(`${item.sourcePhrase} ${item.clusterTitle}`);
return new Set(
[...sourceStems].filter((stem) =>
isProtectedKeywordCleaningFacetStem(stem, guard.approvedAnchorCount, guard.stemDocumentCounts)
)
);
}
function getKeywordCleaningSemanticDowngradeReason(item: KeywordCleaningItem, guard: KeywordCleaningFacetGuard) {
const phraseStems = getKeywordCleaningFacetStems(item.phrase);
const sourceProtectedStems = getProtectedSourceFacetStems(item, guard);
const missingSourceStems = [...sourceProtectedStems].filter((stem) => !phraseStems.has(stem));
if (sourceProtectedStems.size > 0 && missingSourceStems.length === sourceProtectedStems.size) {
return `Wordstat related потерял protected-признак исходного якоря (${missingSourceStems.join(", ")}).`;
}
const dominantProtectedStems = guard.dominantProtectedStems.map((item) => item.stem);
const missingDominantStems = dominantProtectedStems.filter((stem) => !phraseStems.has(stem));
if (dominantProtectedStems.length > 0 && missingDominantStems.length === dominantProtectedStems.length) {
return `Фраза не содержит доминантный protected-смысл approved anchors (${missingDominantStems.join(", ")}).`;
}
return null;
}
function enforceKeywordCleaningSemanticGuard(
market: MarketEnrichmentContract,
items: KeywordCleaningItem[]
): KeywordCleaningItem[] {
const guard = buildKeywordCleaningFacetGuard(market);
if (guard.approvedAnchorCount === 0) {
return items;
}
return items.map((item) => {
if (item.decision !== "use" && item.decision !== "support") {
return item;
}
const semanticDowngradeReason = getKeywordCleaningSemanticDowngradeReason(item, guard);
if (!semanticDowngradeReason) {
return item;
}
return {
...item,
blockers: unique([...item.blockers, "semantic_facet_loss"]),
confidence: Math.min(item.confidence, 0.62),
decision: "risky",
reason: `${semanticDowngradeReason} Backend guard понизил фразу в risky: частотность не перекрывает потерю смысла.`
};
});
}
function getTargetPath(seed: SeedQueueItem, briefPhraseMap: Map<string, string>, briefClusterMap: Map<string, string>) {
const directTarget = briefPhraseMap.get(normalizePhrase(seed.phrase));
@ -704,7 +953,11 @@ function buildReadiness(items: KeywordCleaningItem[], lanes: KeywordCleaningCont
articleCount: lanes.article.length,
exactEvidenceCount: items.filter((item) => item.frequency !== null).length,
keywordMapCandidateCount: items.filter(
(item) => item.decision !== "trash" && item.evidenceStatus === "collected" && item.frequency !== null && item.targetPath
(item) =>
(item.decision === "use" || item.decision === "support") &&
item.evidenceStatus === "collected" &&
item.frequency !== null &&
item.targetPath
).length,
missingPageBindingCount: items.filter((item) => item.targetPath === null && item.decision !== "trash").length,
riskyCount: lanes.risky.length,
@ -734,6 +987,9 @@ function buildKeywordCleaningModelTask(
],
decisionPolicy: {
articleDoesNotEnterRewrite: true,
broadFrequencyDoesNotOverrideSemanticFit: true,
demoteRelatedWhenDominantFacetMissing: true,
preserveApprovedAnchorFacets: true,
riskyWhenNoPageBinding: true,
trashWhenOffContext: true,
useRequiresExactEvidence: true
@ -777,9 +1033,11 @@ function buildKeywordCleaningModelTask(
providerMode: "model_provider_contract",
schemaVersion: "seo-keyword-cleaning-task.v1",
semanticRunId: contract.semanticRunId,
semanticGuard: buildKeywordCleaningSemanticGuard(contract),
stopConditions: [
"Не отправлять trash в keyword map. Risky/article exact-фразы можно только предсортировать в Stage 3, но не считать approved без пользователя.",
"Не придумывать спрос без Wordstat/SERP evidence.",
"Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря или доминантный смысл approved anchors.",
"Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.",
"Не сохранять rewrite/apply decisions."
],
@ -824,7 +1082,7 @@ function buildNextActions(contract: Omit<KeywordCleaningContract, "nextActions">
function buildFallbackContract(projectId: string, market: MarketEnrichmentContract): KeywordCleaningContract {
const modelTask = buildKeywordCleaningModelTask(projectId, market);
const items = classifyItems(market, getBaseItems(market));
const items = enforceKeywordCleaningSemanticGuard(market, classifyItems(market, getBaseItems(market)));
const lanes = buildLanes(items);
const readiness = buildReadiness(items, lanes);
const contractWithoutActions: Omit<KeywordCleaningContract, "nextActions"> = {
@ -990,13 +1248,34 @@ export async function getKeywordCleaningContract(projectId: string): Promise<Key
const latest = await getLatestAlignedKeywordCleaning(projectId, market.semanticRunId, market.wordstat.latestJob?.id ?? null);
return latest
? {
...latest,
modelTask: fallbackContract.modelTask,
sourceTaskId: fallbackContract.sourceTaskId
if (!latest) {
return fallbackContract;
}
: fallbackContract;
const items = enforceKeywordCleaningSemanticGuard(market, latest.items);
const lanes = buildLanes(items);
const readiness = buildReadiness(items, lanes);
return {
...latest,
items,
lanes,
modelTask: fallbackContract.modelTask,
nextActions: buildNextActions({
...latest,
items,
lanes,
modelTask: fallbackContract.modelTask,
readiness,
sourceTaskId: fallbackContract.sourceTaskId
}),
readiness,
sourceTaskId: fallbackContract.sourceTaskId,
summary:
items.length > 0
? `Очищено ${items.length} фраз: ${readiness.useCount} use, ${readiness.supportCount} support, ${readiness.articleCount} article, ${readiness.riskyCount} risky, ${readiness.trashCount} trash.`
: "Нет фраз для keyword cleaning."
};
}
export async function saveKeywordCleaningManual(
@ -1051,7 +1330,7 @@ export async function saveKeywordCleaningFromModelProvider(
const alignedOutput: KeywordCleaningContract = {
...output,
items: mergeModelProviderItemsWithMarketEvidence(output.items, market),
items: enforceKeywordCleaningSemanticGuard(market, mergeModelProviderItemsWithMarketEvidence(output.items, market)),
modelTask,
projectOntologyVersion: market.projectOntologyVersion,
semanticRunId: market.semanticRunId,

View File

@ -103,30 +103,6 @@ function getKeywordProfileText(item: Pick<KeywordCleaningItem, "clusterTitle" |
return normalizePhrase(`${item.phrase} ${item.clusterTitle}`);
}
function isAiDifferentiatorCandidate(item: Pick<KeywordCleaningItem, "clusterTitle" | "phrase">) {
const text = getKeywordProfileText(item);
return /(^|\s)(ai|ии)(\s|$)/i.test(text) || /(агент|агентн|ассистент|assistant|agentic)/i.test(text);
}
function hasProductLikeHybridSignal(value: string) {
const text = normalizePhrase(value);
const hasProductNoun = /(систем[ауы]?|платформ[ауы]?|сервис|software|product|suite)/i.test(text);
const hasProcessNoun = /(бизнес[-\s]?процесс|bpms?|workflow|процесс[а-яё]*)/i.test(text);
return hasProductNoun && hasProcessNoun;
}
function isBroadArticleBacklogPhrase(value: string) {
const text = normalizePhrase(value);
return /^автоматизаци[яи]\s+бизнес[-\s]?процесс(ов|ы)?$/i.test(text);
}
function isHybridSecondaryCandidate(item: KeywordCleaningItem) {
return item.frequency !== null && hasProductLikeHybridSignal(item.phrase) && !isAiDifferentiatorCandidate(item);
}
function isReviewedOntology(version: MarketEnrichmentContract["projectOntologyVersion"]) {
return (
Boolean(version) &&
@ -214,6 +190,17 @@ type SemanticFacetGuard = {
stemDocumentCounts: Map<string, number>;
};
type SemanticFitAssessment = {
label: "lost_project_facet" | "lost_source_facet" | "strong" | "weak";
matchedProjectStems: string[];
matchedSourceStems: string[];
missingProjectStems: string[];
missingSourceStems: string[];
projectCoverage: number | null;
score: number;
sourceCoverage: number | null;
};
function stemKeywordToken(token: string) {
const normalizedToken = token.toLocaleLowerCase("ru-RU").replace(/ё/g, "е");
@ -318,14 +305,62 @@ function hasSemanticFacetLoss(item: KeywordCleaningItem, guard: SemanticFacetGua
return getMissingSemanticFacetStems(item, guard).length > 0;
}
function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) {
if (guard.dominantProtectedStems.size === 0) {
return false;
function getCoverage(matchedCount: number, totalCount: number) {
return totalCount > 0 ? matchedCount / totalCount : null;
}
function getSemanticFitAssessment(item: KeywordCleaningItem, guard: SemanticFacetGuard): SemanticFitAssessment {
const phraseStems = getKeywordFacetStems(item.phrase);
const sourceStems = item.source === "wordstat_related" ? getProtectedSourceFacetStems(item, guard) : new Set<string>();
const projectStems = guard.dominantProtectedStems;
const matchedSourceStems = [...sourceStems].filter((stem) => phraseStems.has(stem));
const matchedProjectStems = [...projectStems].filter((stem) => phraseStems.has(stem));
const missingSourceStems = [...sourceStems].filter((stem) => !phraseStems.has(stem));
const missingProjectStems = [...projectStems].filter((stem) => !phraseStems.has(stem));
const sourceCoverage = getCoverage(matchedSourceStems.length, sourceStems.size);
const projectCoverage = getCoverage(matchedProjectStems.length, projectStems.size);
const score = Math.min(sourceCoverage ?? 1, projectCoverage ?? 1);
if (sourceCoverage === 0 && sourceStems.size > 0) {
return {
label: "lost_source_facet",
matchedProjectStems,
matchedSourceStems,
missingProjectStems,
missingSourceStems,
projectCoverage,
score,
sourceCoverage
};
}
const phraseStems = getKeywordFacetStems(item.phrase);
if (projectCoverage === 0 && projectStems.size > 0) {
return {
label: "lost_project_facet",
matchedProjectStems,
matchedSourceStems,
missingProjectStems,
missingSourceStems,
projectCoverage,
score,
sourceCoverage
};
}
return [...guard.dominantProtectedStems].every((stem) => !phraseStems.has(stem));
return {
label: score >= 0.5 ? "strong" : "weak",
matchedProjectStems,
matchedSourceStems,
missingProjectStems,
missingSourceStems,
projectCoverage,
score,
sourceCoverage
};
}
function missesDominantProjectFacet(item: KeywordCleaningItem, guard: SemanticFacetGuard) {
return getSemanticFitAssessment(item, guard).label === "lost_project_facet";
}
function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) {
@ -335,13 +370,16 @@ function getKeywordPriorityWeight(priority: KeywordCleaningItem["priority"]) {
}
function canModelRouteKeyword(item: KeywordCleaningItem, guard?: SemanticFacetGuard) {
const semanticFit = guard ? getSemanticFitAssessment(item, guard) : null;
return (
(item.decision === "use" || item.decision === "support") &&
item.evidenceStatus === "collected" &&
item.frequency !== null &&
Boolean(item.projectOntologyVersionId) &&
Boolean(item.wordstatResultId) &&
Boolean(item.targetPath) &&
(!guard || (!hasSemanticFacetLoss(item, guard) && !missesDominantProjectFacet(item, guard)))
(!semanticFit || (semanticFit.label === "strong" && semanticFit.score >= 0.5))
);
}
@ -374,14 +412,14 @@ function getKeywordRole(item: KeywordCleaningItem, indexInTarget: number, guard:
return "support";
}
if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") {
return "secondary";
}
if (isClarifyingKeywordCandidate(item)) {
return "differentiator";
}
if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") {
return "secondary";
}
if (item.priority === "high" || item.priority === "medium") {
return "secondary";
}
@ -439,6 +477,13 @@ function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleanin
return rightPriorityWeight - leftPriorityWeight;
}
const leftSemanticFit = getSemanticFitAssessment(left, semanticFacetGuard).score;
const rightSemanticFit = getSemanticFitAssessment(right, semanticFacetGuard).score;
if (leftSemanticFit !== rightSemanticFit) {
return rightSemanticFit - leftSemanticFit;
}
const leftFrequency = left.frequency ?? -1;
const rightFrequency = right.frequency ?? -1;
@ -514,15 +559,15 @@ function getKeywordMapItemReason(
}
if (role === "validate" && hasSemanticFacetLoss(item, semanticFacetGuard)) {
return "Wordstat related потерял доминантный признак исходного якоря: оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта.";
const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard);
return `Wordstat related потерял protected-признак исходного якоря (${semanticFit.missingSourceStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не вытеснил смысл проекта.`;
}
if (role === "validate" && missesDominantProjectFacet(item, semanticFacetGuard)) {
return "Фраза не содержит доминантный смысл из подтверждённой очереди якорей: оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически.";
}
const semanticFit = getSemanticFitAssessment(item, semanticFacetGuard);
if (isBroadArticleBacklogPhrase(item.phrase) && role === "validate") {
return "Широкая head-фраза уходит в article/backlog: не делаем её прямой основной ставкой текущей посадочной без отдельного SERP page-type решения.";
return `Фраза не содержит доминантный protected-смысл из подтверждённой очереди (${semanticFit.missingProjectStems.join(", ")}): оставляем на ручной разбор, чтобы широкий спрос не стал ядром автоматически.`;
}
if (item.decision === "article") {