diff --git a/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts b/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts index 0f295ad..eb0b315 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts @@ -3,6 +3,7 @@ import { pool } from "../db/client.js"; import { getAnchorReviewContract } from "./anchorReview.js"; import { getKeywordCleaningByModelTaskRun, getKeywordCleaningContract } from "./keywordCleaning.js"; import { getMarketEnrichmentContract, runMarketEnrichment } from "../market/marketEnrichment.js"; +import { getMarketFrontierDiscoveryContract } from "../market/marketFrontierDiscovery.js"; import { getSeoModelProviderStatusContract, runSeoModelTask } from "../modelProvider/seoModelTaskRuns.js"; import { updateProjectOntologyApproval } from "../ontology/projectOntologyRepository.js"; @@ -325,6 +326,30 @@ async function ensureMarketFrontierDiscoveryReady( return { contract, ready: true }; } + const currentDiscovery = await getMarketFrontierDiscoveryContract(projectId); + + if ( + currentDiscovery.semanticRunId && + currentDiscovery.provider.mode !== "deterministic_fallback" && + currentDiscovery.probeSeeds.length > 0 + ) { + const ready = setWorkflowStatus( + setStepStatus(contract, "demand_collection", "running", { + detail: + `Используем сохранённый market frontier discovery: ${currentDiscovery.probeSeeds.length} probes уже готовы, повторный AI Workspace вызов не нужен.` + }), + "running", + { + activeStepId: "demand_collection", + reason: null, + summary: "Используем сохранённый market frontier discovery и продолжаем Wordstat-добор." + } + ); + await saveWorkflow(ready, "running"); + + return { contract: ready, ready: true }; + } + const modelTaskRun = await runSeoModelTask(projectId, { providerId: "codex_workspace", taskType: "seo.market_frontier_discovery" diff --git a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts index 21205f0..029d89c 100644 --- a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts +++ b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts @@ -300,6 +300,8 @@ const EXPANSION_QUERY_BLOCKLIST = [ /(^|\s)(задач|проверок)\.$/i, /(^|\s)(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)\s+(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)$/i, /^(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)\s+(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)(\s|$)/i, + /^(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)(\s+и\s+(llm|нейросет[а-яёa-z0-9-]*|модел[а-яёa-z0-9-]*))?$/i, + /^(данные\s+и\s+аналитика|агент\s+ассистент|ai[-\s]?ассистенты\s+и\s+агентная\s+автоматизация)$/i, /^(автоматизац[а-яёa-z0-9-]*|применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(выбор|оценк[аи]|ценност[ьи])$/i, /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(данн[а-яё]*|процесс[а-яё]*|задач[а-яё]*)$/i, /[.!?:]/, @@ -311,7 +313,8 @@ const DEMAND_COLLECTION_MAX_BASE_RUNS = 1; const DEMAND_COLLECTION_MAX_MARKET_WIDE_RUNS = 1; const DEMAND_COLLECTION_CONTEXTUAL_PROBE_BUDGET = 8; const DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET = 12; -const DEMAND_COLLECTION_MARKET_WIDE_ANCHOR_PROBE_BUDGET = 8; +const DEMAND_COLLECTION_MARKET_WIDE_ANCHOR_PROBE_BUDGET = 2; +const DEMAND_COLLECTION_MARKET_WIDE_FRONTIER_PROBE_BUDGET = 8; const WORDSTAT_PROBE_MAX_PER_PATTERN_FAMILY = 4; const WORDSTAT_PROBE_MAX_PER_CLUSTER = 8; @@ -480,8 +483,8 @@ function getMarketRoleProbeWeight(role: AnchorReviewWordstatQueueItem["marketRol } function getProbeSourceWeight(source: WordstatProbeSourceKind) { - if (source === "approved_anchor") return 34; - if (source === "market_frontier_discovery") return 28; + if (source === "market_frontier_discovery") return 42; + if (source === "approved_anchor") return 28; if (source === "commercial_demand") return 20; if (source === "market_anchor_expansion") return 14; return 10; @@ -494,11 +497,43 @@ function hasCommercialProbeSignal(phrase: string) { } function hasMarketQualifier(phrase: string) { - return /(^|\s)(ии|ai|корпоративн[а-яё]*|enterprise|бизнес[а-яё]*|промышленн[а-яё]*|компани[яйи]|предприяти[еяй]|малого\s+бизнеса|отрасл[а-яё]*|закуп|тендер|документооборот|учет|производств|строительств)(\s|$)/i.test( + return /(^|\s)(ии|ai|корпоративн[а-яё]*|enterprise|бизнес[а-яё]*|промышленн[а-яё]*|компани[яйи]|предприяти[еяй]|мал[а-яё]*\s+бизнес[а-яё]*|отрасл[а-яё]*|закуп|тендер|документооборот|учет|производств|строительств|беспилот[а-яё]*|бпла|рутинн[а-яё]*\s+задач[а-яё]*)(\s|$)/i.test( phrase ); } +const NON_PRODUCT_PROBE_TARIFF_CONTEXT_PATTERN = + /(^|\s)(банк[а-яё]*|банковск[а-яё]*|счет[а-яё]*|счёт[а-яё]*|ип|рко|эквайринг|интернет|связ[ьи]|мобил[а-яё]*|телефон[а-яё]*|оператор[а-яё]*|альфа|втб|сбер|сбербанк|тинькофф|мегафон|билайн|мтс|tele2|360)(\s|$)/i; +const PRODUCT_PROBE_TARIFF_QUALIFIER_PATTERN = + /(^|\s)(ai|ии|llm|нейросет[а-яё]*|искусственн[а-яё]*\s+интеллект|агент[а-яё]*|ассистент[а-яё]*|автоматизац[а-яё]*|платформ[а-яё]*|сервис[а-яё]*|crm|erp|bpm|workflow|интеграц[а-яё]*|внедрен[а-яё]*|разработк[а-яё]*|корпоративн[а-яё]*|enterprise)(\s|$)/i; +const BUSINESS_PLANNING_PROBE_PATTERNS = [ + /(^|\s)(бизнес[-\s]?план[а-яё]*|создани[ея]\s+бизнес[а-яё]*|открыти[ея]\s+бизнес[а-яё]*)(\s|$)/i, + /(^|\s)(бизнес[-\s]?модел[а-яё]*|модел[ьи]\s+для\s+бизнес[а-яё]*|стратеги[яи]\s+для\s+бизнес[-\s]?модел[а-яё]*)(\s|$)/i +]; + +function hasNonProductProbeIntentMismatch(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + const isTariffPhrase = /(^|\s)(тариф[а-яё]*|расценк[а-яё]*)(\s|$)/i.test(normalizedPhrase); + + if (BUSINESS_PLANNING_PROBE_PATTERNS.some((pattern) => pattern.test(normalizedPhrase))) { + return true; + } + + if (!isTariffPhrase) { + return false; + } + + if (NON_PRODUCT_PROBE_TARIFF_CONTEXT_PATTERN.test(normalizedPhrase)) { + return true; + } + + return ( + /(^|\s)для\s+(?:мал[а-яё]*\s+|средн[а-яё]*\s+|крупн[а-яё]*\s+)?(бизнес[а-яё]*|предприяти[а-яё]*|компани[а-яё]*)(\s|$)/i.test( + normalizedPhrase + ) && !PRODUCT_PROBE_TARIFF_QUALIFIER_PATTERN.test(normalizedPhrase) + ); +} + function getWeakGenericProbeReason(phrase: string) { const normalizedPhrase = normalizePhrase(phrase); const words = wordCount(normalizedPhrase); @@ -526,12 +561,12 @@ function getProbeMarketQualityBoost(phrase: string) { (hasMarketQualifier(normalizedPhrase) ? 16 : 0) + (/(^|\s)(купить|заказать|стоимость|цена|тариф|поставщик|подрядчик)(\s|$)/i.test(normalizedPhrase) ? 14 : 0) + (/(^|\s)(для\s+бизнеса|для\s+компани[йи]|для\s+предприяти[яй]|корпоративн[а-яё]*)(\s|$)/i.test(normalizedPhrase) ? 12 : 0) + - (/(^|\s)(промышленн[а-яё]*|строительств|закуп|тендер|документооборот|учет|производств)(\s|$)/i.test(normalizedPhrase) ? 10 : 0) + (/(^|\s)(промышленн[а-яё]*|строительств|закуп|тендер|документооборот|учет|производств|беспилот[а-яё]*|бпла|мал[а-яё]*\s+бизнес[а-яё]*|рутинн[а-яё]*\s+задач[а-яё]*)(\s|$)/i.test(normalizedPhrase) ? 14 : 0) ); } function getProbeGenericPenalty(phrase: string) { - return getWeakGenericProbeReason(phrase) ? -34 : 0; + return getWeakGenericProbeReason(phrase) ? -80 : 0; } function getProbePatternFamily(phrase: string) { @@ -592,6 +627,12 @@ function hasAwkwardWordstatProbeSyntax(phrase: string) { /^(заказать|купить|разработк[аи]|создани[ея]|внедрени[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+enterprise\s+(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*|решени[ея])/i.test( normalizedPhrase ) || + /^(тариф|цена|стоимость)\s+(ops|hub|engine|workspace|platform|cloud)(\s|$)/i.test(normalizedPhrase) || + /(^|\s)(агент|ассистент|роль|workflow|автоматизац[а-яё]*|систем[а-яё]*|платформ[а-яё]*)\s+(ops|hub|engine)(\s|$)/i.test( + normalizedPhrase + ) || + /^(ops|hub|engine)\s+для\s+/i.test(normalizedPhrase) || + /^(операционн[а-яё]*\s+клиент|клиент\s+операционн[а-яё]*|доступа\s+для\s+)/i.test(normalizedPhrase) || /^(внедрени[ея]|интеграц(?:ия|ии|ию|ией|ие)|разработк[аи]|создани[ея])\s+(сервис[а-яё]*|платформ[а-яё]*|систем[а-яё]*|решени[яе])\s+(ии|ai)\s+(ассистент|агент|платформа|система|сервис)(\s|$)/i.test( normalizedPhrase ) || @@ -631,6 +672,7 @@ function isPlausibleWordstatProbePhrase(phrase: string) { !["n/a", "na", "null", "undefined", "нет данных", "не применимо"].includes(normalizedPhrase) && !/[{}[\]<>#$%^*=~`/|]/.test(normalizedPhrase) && !hasAwkwardWordstatProbeSyntax(normalizedPhrase) && + !hasNonProductProbeIntentMismatch(normalizedPhrase) && !EXPANSION_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedPhrase)) && !hasUnsafeExpansionLatinToken(normalizedPhrase) ); @@ -663,6 +705,7 @@ function scoreWordstatProbeCandidate(candidate: WordstatProbeCandidate) { (hasCommercialProbeSignal(normalizedPhrase) ? 12 : 0) + getProbeMarketQualityBoost(normalizedPhrase) + getProbeGenericPenalty(normalizedPhrase) + + (hasNonProductProbeIntentMismatch(normalizedPhrase) ? -120 : 0) + (candidate.normalizedFrom?.length ? 4 : 0) ); } @@ -852,21 +895,44 @@ function selectMarketWideWordstatProbeSeeds( blockedPhrases ); const usedAnchorPhrases = new Set(anchorProbeSeeds.map((seed) => normalizePhrase(seed.phrase))); - const blockedAndUsedPhrases = new Set([...blockedPhrases, ...usedAnchorPhrases]); - const remainingBudget = DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET - anchorProbeSeeds.length; const anchorFamilyCounts = getWordstatProbeFamilyCounts(anchorProbeSeeds); + const frontierBudget = Math.min( + DEMAND_COLLECTION_MARKET_WIDE_FRONTIER_PROBE_BUDGET, + Math.max(0, DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET - anchorProbeSeeds.length) + ); + const frontierProbeSeeds = + frontierBudget > 0 + ? selectWordstatProbeSeeds( + candidates.filter((candidate) => candidate.probeSource === "market_frontier_discovery"), + wordstat, + frontierBudget, + new Set([...blockedPhrases, ...usedAnchorPhrases]), + { initialFamilyCounts: anchorFamilyCounts, maxPerFamily: 2 } + ) + : []; + const usedFrontierPhrases = new Set(frontierProbeSeeds.map((seed) => normalizePhrase(seed.phrase))); + const usedFamilyCounts = getWordstatProbeFamilyCounts([...anchorProbeSeeds, ...frontierProbeSeeds]); + const remainingBudget = + DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET - anchorProbeSeeds.length - frontierProbeSeeds.length; const derivedProbeSeeds = remainingBudget > 0 ? selectWordstatProbeSeeds( - candidates.filter((candidate) => candidate.probeSource !== "approved_anchor"), - wordstat, - remainingBudget, - blockedAndUsedPhrases, - { initialFamilyCounts: anchorFamilyCounts } - ) + candidates.filter( + (candidate) => + candidate.probeSource !== "approved_anchor" && + candidate.probeSource !== "market_frontier_discovery" + ), + wordstat, + remainingBudget, + new Set([...blockedPhrases, ...usedAnchorPhrases, ...usedFrontierPhrases]), + { initialFamilyCounts: usedFamilyCounts } + ) : []; - return interleaveWordstatSeedGroups([anchorProbeSeeds, derivedProbeSeeds], DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET); + return interleaveWordstatSeedGroups( + [frontierProbeSeeds, anchorProbeSeeds, derivedProbeSeeds], + DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET + ); } function buildAnchorWordstatProbeSeeds(anchorReview: AnchorReviewContract, suppressedPhrases: Set) { diff --git a/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts b/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts index eaf8176..30c21df 100644 --- a/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts +++ b/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts @@ -656,6 +656,144 @@ function getAnchorSeedPhrases(anchorReview: AnchorReviewContract) { })); } +const SEMANTIC_APPLICATION_FRONTIER_RULES = [ + { + evidencePattern: /bim|digital\s+twin|цифров[а-яё]*\s+двойн|строител|инженер|3d|step|stl|las/i, + phrases: [ + "применение ии в строительстве", + "ии в строительстве", + "цифровой двойник в строительстве" + ], + title: "Строительство и инженерные объекты" + }, + { + evidencePattern: /документооборот|договор|регламент|согласован|процесс|тендер|закуп/i, + phrases: [ + "ии в документообороте", + "автоматизация документооборота с ии", + "ии автоматизация бизнес процессов", + "автоматизация рутинных задач с ии" + ], + title: "Документооборот и бизнес-процессы" + }, + { + evidencePattern: /промышлен|производств|iot|телеметр|робот|беспилот|бпла/i, + phrases: [ + "внедрение ии в промышленность", + "ии в промышленности", + "ии для производства", + "ии для беспилотников" + ], + title: "Промышленность и автономные системы" + }, + { + evidencePattern: /разработк|api|приложен|код|агент|ассистент|llm/i, + phrases: [ + "разработка ии агентов", + "разработка ai ассистентов", + "создание ии агентов для бизнеса" + ], + title: "Разработка ИИ-агентов" + }, + { + evidencePattern: /enterprise|корпоратив|контур|on[-\s]?prem|private\s+cloud|предприяти/i, + phrases: [ + "внедрение ии в бизнес", + "ии для предприятия", + "корпоративные ии агенты" + ], + title: "Корпоративное внедрение" + }, + { + evidencePattern: /1с|crm|erp|интеграц|api|данн|аналитик|хранилищ/i, + phrases: [ + "ии интеграция с 1с", + "ии автоматизация crm", + "автоматизация отчетности с ии" + ], + title: "Интеграции и данные" + } +]; + +function getSemanticApplicationSeedPhrases(analysis: SemanticAnalysisRun, anchorReview: AnchorReviewContract) { + const semanticText = [ + ...analysis.clusters.flatMap((cluster) => [ + cluster.title, + cluster.targetIntent, + ...cluster.matchedTerms, + ...cluster.queryExamples + ]), + ...anchorReview.wordstatQueue.flatMap((anchor) => [ + anchor.phrase, + anchor.clusterTitle, + anchor.intentType, + anchor.marketRole, + ...anchor.normalizedFrom + ]) + ].join(" "); + const hasAiAutomationContext = + /(^|\s)(ии|ai|llm|агент|ассистент|нейросет|искусственн[а-яё]*\s+интеллект|автоматизац)(\s|$)/i.test(semanticText); + + if (!hasAiAutomationContext) { + return []; + } + + const seeds: Array<{ + phrase: string; + title: string; + priority: "high" | "medium" | "low"; + sourceText: string; + evidenceRefs: string[]; + }> = [ + { + phrase: "ии для бизнеса", + title: "Широкий B2B AI спрос", + priority: "high", + sourceText: "Semantic clusters show AI/agent/automation context; broad B2B probe checks market language before deeper verticals.", + evidenceRefs: ["semanticApplication:ai-business"] + }, + { + phrase: "ии агенты для бизнеса", + title: "AI-агенты для бизнеса", + priority: "high", + sourceText: "Semantic clusters contain agent/assistant automation; probe validates how buyers phrase agent demand.", + evidenceRefs: ["semanticApplication:ai-agents-business"] + }, + { + phrase: "автоматизация бизнес процессов с ии", + title: "AI-автоматизация процессов", + priority: "high", + sourceText: "Semantic clusters contain process/workflow automation; probe validates process automation language.", + evidenceRefs: ["semanticApplication:ai-process-automation"] + }, + { + phrase: "ии для малого бизнеса", + title: "Малый бизнес как B2B-сегмент", + priority: "medium", + sourceText: "Market-wide mode should test SME language when product value is framed as business automation.", + evidenceRefs: ["semanticApplication:small-business"] + } + ]; + + for (const rule of SEMANTIC_APPLICATION_FRONTIER_RULES) { + if (!rule.evidencePattern.test(semanticText)) { + continue; + } + + for (const phrase of rule.phrases) { + seeds.push({ + phrase, + title: rule.title, + priority: "high", + sourceText: `Semantic cluster evidence matched ${rule.title}; market-wide mode should test this application frontier.`, + evidenceRefs: [`semanticApplication:${toId(rule.title)}`] + }); + } + } + + return seeds.filter((seed) => isPlausibleProbePhrase(seed.phrase)); +} + function getFingerprintProbePhrases(fingerprint: QueryLanguageFingerprint, anchorReview: AnchorReviewContract) { const anchorTerms = unique( anchorReview.wordstatQueue @@ -679,6 +817,7 @@ function getFingerprintProbePhrases(fingerprint: QueryLanguageFingerprint, ancho } function buildFallbackFrontiers(input: { + analysis: SemanticAnalysisRun; anchorReview: AnchorReviewContract; commercialDemand: SeoCommercialDemandContract | null; fingerprint: QueryLanguageFingerprint; @@ -687,6 +826,7 @@ function buildFallbackFrontiers(input: { const covered = new Set(input.memory.coveredPhrases.map(normalizePhrase)); const suppressed = new Set(input.memory.suppressedPhrases.map(normalizePhrase)); const seedInputs = [ + ...getSemanticApplicationSeedPhrases(input.analysis, input.anchorReview), ...getCommercialSeedPhrases(input.commercialDemand), ...getAnchorSeedPhrases(input.anchorReview), ...getFingerprintProbePhrases(input.fingerprint, input.anchorReview) @@ -799,6 +939,33 @@ function buildProbeSeedsFromFrontiers(frontiers: MarketFrontierDiscoveryContract return seeds.slice(0, 48); } +function mergeMarketFrontierProbeSeeds( + primarySeeds: MarketFrontierProbeSeed[], + fallbackSeeds: MarketFrontierProbeSeed[], + memory: MarketResearchMemory +) { + const blocked = new Set([...memory.coveredPhrases, ...memory.suppressedPhrases].map(normalizePhrase)); + const seen = new Set(); + const merged: MarketFrontierProbeSeed[] = []; + + for (const seed of [...primarySeeds, ...fallbackSeeds]) { + const normalizedPhrase = normalizePhrase(seed.phrase); + + if (!isPlausibleProbePhrase(normalizedPhrase) || blocked.has(normalizedPhrase) || seen.has(normalizedPhrase)) { + continue; + } + + seen.add(normalizedPhrase); + merged.push({ + ...seed, + normalizedPhrase, + phrase: normalizedPhrase + }); + } + + return merged.slice(0, 72); +} + function buildModelTaskContract(input: { analysis: SemanticAnalysisRun; anchorReview: AnchorReviewContract; @@ -955,6 +1122,7 @@ function buildFallbackContract(input: { : null; const frontiers = input.analysis ? buildFallbackFrontiers({ + analysis: input.analysis, anchorReview: input.anchorReview, commercialDemand: input.commercialDemand, fingerprint: input.fingerprint, @@ -1058,6 +1226,7 @@ function normalizeModelOutput( const probeSeeds = (Array.isArray(output.probeSeeds) ? output.probeSeeds : []) .map((seed) => normalizeProbeSeed(seed, frontierById, fallback.marketMemory)) .filter((seed): seed is MarketFrontierProbeSeed => Boolean(seed)); + const mergedProbeSeeds = mergeMarketFrontierProbeSeeds(probeSeeds, fallback.probeSeeds, fallback.marketMemory); const contractWithoutActions: Omit = { ...output, analystReview: output.analystReview ?? getDefaultAnalystReview(), @@ -1071,10 +1240,10 @@ function normalizeModelOutput( marketMemory: fallback.marketMemory, queryLanguageFingerprint: output.queryLanguageFingerprint ?? fallback.queryLanguageFingerprint, frontiers, - probeSeeds, + probeSeeds: mergedProbeSeeds, modelTask: fallback.modelTask, rejectedPatterns: Array.isArray(output.rejectedPatterns) ? output.rejectedPatterns : fallback.rejectedPatterns, - summary: output.summary || `Market frontier discovery вернул ${frontiers.length} frontiers и ${probeSeeds.length} Wordstat probes.` + summary: output.summary || `Market frontier discovery вернул ${frontiers.length} frontiers и ${mergedProbeSeeds.length} Wordstat probes.` }; return { @@ -1140,6 +1309,7 @@ export async function getMarketFrontierDiscoveryContract(projectId: string): Pro ...latest, marketMemory: fallback.marketMemory, modelTask: fallback.modelTask, + probeSeeds: mergeMarketFrontierProbeSeeds(latest.probeSeeds, fallback.probeSeeds, fallback.marketMemory), queryLanguageFingerprint: latest.queryLanguageFingerprint ?? fallback.queryLanguageFingerprint } : fallback; }