From a1d1a4980fe2f508e284e3e0c7e8ca1041c1b5d8 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Sun, 5 Jul 2026 11:07:13 +0300 Subject: [PATCH] feat(seo): orchestrate market demand discovery --- seo_mode/seo_mode/.env.example | 2 +- .../server/src/analysis/semanticAnalysis.ts | 280 ++- .../src/business/seoBusinessSynthesis.ts | 1047 +++++++++++ .../src/commercial/seoCommercialDemand.ts | 732 ++++++++ seo_mode/seo_mode/server/src/config/env.ts | 2 +- .../src/contextReview/seoContextReview.ts | 345 +++- .../server/src/keywords/anchorReview.ts | 270 ++- .../src/keywords/keywordAnalysisWorkflow.ts | 768 ++++++++ .../server/src/keywords/keywordCleaning.ts | 164 +- .../src/keywords/keywordContextSnapshots.ts | 261 +++ .../src/keywords/keywordIntentGuards.ts | 108 ++ .../server/src/keywords/keywordMap.ts | 470 ++++- .../server/src/market/marketEnrichment.ts | 1629 ++++++++++++++++- .../src/market/marketFrontierDiscovery.ts | 1206 ++++++++++++ .../server/src/market/wordstatProvider.ts | 69 +- .../server/src/market/wordstatRepository.ts | 74 +- .../src/modelContext/seoModelContext.ts | 46 +- .../src/modelProvider/aiWorkspaceBridge.ts | 63 +- .../modelProvider/modelProviderRegistry.ts | 3 + .../src/modelProvider/seoModelTaskRuns.ts | 218 ++- .../src/normalization/seoNormalization.ts | 1128 +++++++++++- .../server/src/ontology/projectOntology.ts | 62 +- .../seo_mode/server/src/projects/routes.ts | 209 ++- .../src/settings/externalServiceSettings.ts | 8 +- .../server/src/skills/seoSkillRegistry.ts | 77 +- 25 files changed, 8929 insertions(+), 312 deletions(-) create mode 100644 seo_mode/seo_mode/server/src/business/seoBusinessSynthesis.ts create mode 100644 seo_mode/seo_mode/server/src/commercial/seoCommercialDemand.ts create mode 100644 seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts create mode 100644 seo_mode/seo_mode/server/src/keywords/keywordContextSnapshots.ts create mode 100644 seo_mode/seo_mode/server/src/keywords/keywordIntentGuards.ts create mode 100644 seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts diff --git a/seo_mode/seo_mode/.env.example b/seo_mode/seo_mode/.env.example index ff4d590..804615d 100644 --- a/seo_mode/seo_mode/.env.example +++ b/seo_mode/seo_mode/.env.example @@ -19,7 +19,7 @@ WORDSTAT_PROVIDER=disabled # WORDSTAT_REGION_IDS=225 # WORDSTAT_DEVICES=DEVICE_ALL # WORDSTAT_NUM_PHRASES=50 -# WORDSTAT_MAX_SEEDS_PER_RUN=40 +# WORDSTAT_MAX_SEEDS_PER_RUN=12 SERP_PROVIDER=disabled # SERP_API_BASE_URL=https://serp-provider.internal diff --git a/seo_mode/seo_mode/server/src/analysis/semanticAnalysis.ts b/seo_mode/seo_mode/server/src/analysis/semanticAnalysis.ts index cbdf761..fa9070a 100644 --- a/seo_mode/seo_mode/server/src/analysis/semanticAnalysis.ts +++ b/seo_mode/seo_mode/server/src/analysis/semanticAnalysis.ts @@ -68,6 +68,37 @@ type ContentDocument = { type PageScope = "indexable_page" | "template_block" | "admin_page" | "technical_page"; +type TextCorpusChunk = { + charEnd: number; + charStart: number; + chunkIndex: number; + documentId: string; + id: string; + kind: ContentDocument["kind"]; + scope: ContentDocument["scope"]; + selected: boolean; + sourcePath: string; + text: string; + title: string; + totalChunks: number; + urlPath: string | null; + usedForCoverage: boolean; +}; + +type TextCorpusDocument = { + chunkIds: string[]; + id: string; + kind: ContentDocument["kind"]; + scope: ContentDocument["scope"]; + selected: boolean; + sourcePath: string; + textCharCount: number; + title: string; + urlPath: string | null; + usedForCoverage: boolean; + wordCount: number; +}; + type AnalysisSignal = { id: string; level: "ok" | "warning" | "problem"; @@ -272,6 +303,19 @@ export type SemanticAnalysisOutput = { wordCount: number; matchedClusters: string[]; }>; + textCorpus: { + schemaVersion: "seo-text-corpus.v1"; + extraction: { + chunkCount: number; + documentCount: number; + mode: "clean_visible_text"; + sourceDocumentCount: number; + totalChars: number; + totalWords: number; + }; + documents: TextCorpusDocument[]; + chunks: TextCorpusChunk[]; + }; legacyFindings: Array<{ markerId: string; label: string; @@ -484,6 +528,11 @@ const RUSSIAN_STOP_WORDS = new Set([ "assets", "attr", "avif", + "accesscard", + "bodyhtml", + "bottominfo", + "card", + "cards", "class", "collection", "content", @@ -492,16 +541,22 @@ const RUSSIAN_STOP_WORDS = new Set([ "data", "features", "footer", + "form", "html", "image", "images", + "introhtml", "json", "heading", "main", "media", "muted", + "node", "site", "src", + "static", + "staticelements", + "target", "text", "txt", "webp", @@ -537,6 +592,8 @@ const CTA_PATTERNS = [ "заказать" ]; +const TEXT_CORPUS_CHUNK_CHARS = 6000; + function toIsoDate(value: Date | null) { return value ? value.toISOString() : null; } @@ -626,12 +683,113 @@ function visibleTextFromHtml(html: string) { .replace(//gi, " ") .replace(//gi, " ") .replace(//gi, " ") + .replace(/<(br|hr)\b[^>]*>/gi, "\n") + .replace(/<\/(article|aside|blockquote|dd|details|dialog|div|dl|dt|figcaption|figure|footer|form|h[1-6]|header|li|main|nav|ol|p|section|summary|table|tbody|td|tfoot|th|thead|tr|ul)>/gi, "\n") .replace(/<[^>]+>/g, " ") ) - .replace(/\s+/g, " ") + .split(/\n+/) + .map((line) => line.replace(/[ \t\f\v]+/g, " ").trim()) + .filter(Boolean) + .join("\n") .trim(); } +function splitTextCorpusChunks(text: string, maxChars = TEXT_CORPUS_CHUNK_CHARS) { + const chunks: Array<{ charEnd: number; charStart: number; text: string }> = []; + let start = 0; + + while (start < text.length) { + const hardEnd = Math.min(text.length, start + maxChars); + let end = hardEnd; + + if (hardEnd < text.length) { + const windowStart = Math.max(start + Math.floor(maxChars * 0.65), start); + const newlineIndex = text.lastIndexOf("\n", hardEnd); + const spaceIndex = text.lastIndexOf(" ", hardEnd); + const breakIndex = Math.max(newlineIndex, spaceIndex); + + if (breakIndex >= windowStart) { + end = breakIndex; + } + } + + const chunkText = text.slice(start, end).trim(); + + if (chunkText) { + chunks.push({ + charEnd: end, + charStart: start, + text: chunkText + }); + } + + start = end; + while (start < text.length && /\s/.test(text[start] ?? "")) { + start += 1; + } + } + + return chunks; +} + +function buildTextCorpus(documents: ContentDocument[]): SemanticAnalysisOutput["textCorpus"] { + const sourceDocuments = documents.filter((document) => document.text.trim().length > 0); + const corpusDocuments: TextCorpusDocument[] = []; + const chunks: TextCorpusChunk[] = []; + + for (const document of sourceDocuments) { + const documentChunks = splitTextCorpusChunks(document.text); + const chunkIds = documentChunks.map((_, index) => `${document.id}:chunk:${index + 1}`); + + corpusDocuments.push({ + chunkIds, + id: document.id, + kind: document.kind, + scope: document.scope, + selected: document.selected, + sourcePath: document.sourcePath, + textCharCount: document.text.length, + title: document.title, + urlPath: document.urlPath, + usedForCoverage: document.usedForCoverage, + wordCount: document.wordCount + }); + + documentChunks.forEach((chunk, index) => { + chunks.push({ + charEnd: chunk.charEnd, + charStart: chunk.charStart, + chunkIndex: index + 1, + documentId: document.id, + id: chunkIds[index] ?? `${document.id}:chunk:${index + 1}`, + kind: document.kind, + scope: document.scope, + selected: document.selected, + sourcePath: document.sourcePath, + text: chunk.text, + title: document.title, + totalChunks: documentChunks.length, + urlPath: document.urlPath, + usedForCoverage: document.usedForCoverage + }); + }); + } + + return { + schemaVersion: "seo-text-corpus.v1", + extraction: { + chunkCount: chunks.length, + documentCount: corpusDocuments.length, + mode: "clean_visible_text", + sourceDocumentCount: sourceDocuments.length, + totalChars: corpusDocuments.reduce((sum, document) => sum + document.textCharCount, 0), + totalWords: corpusDocuments.reduce((sum, document) => sum + document.wordCount, 0) + }, + documents: corpusDocuments, + chunks + }; +} + function normalizeSearchText(value: string) { return value .toLocaleLowerCase("ru-RU") @@ -1316,7 +1474,7 @@ function buildSeedCandidates(clusters: ReturnType, ontol ? `Кластер важен для ${ontology.brandName}, но в текущем сайте почти не раскрыт.` : "Кластер уже найден в контенте и подходит для проверки частотности."; - return cluster.queryExamples.slice(0, cluster.priority === "high" ? 3 : 2).map((phrase) => ({ + return buildClusterSeedPhrases(cluster, ontology.brandName).slice(0, cluster.priority === "high" ? 4 : 2).map((phrase) => ({ phrase, clusterId: cluster.id, clusterTitle: cluster.title, @@ -1327,6 +1485,122 @@ function buildSeedCandidates(clusters: ReturnType, ontol }); } +function getSeedPhraseWords(value: string) { + return normalizeSearchText(value) + .split(/\s+/) + .map((word) => word.trim()) + .filter(Boolean); +} + +function isSemanticBasisPhrase(value: string, brandName: string) { + const normalizedValue = normalizeSearchText(value); + const words = getSeedPhraseWords(normalizedValue); + const normalizedBrand = normalizeSearchText(brandName); + const brandWords = new Set(getSeedPhraseWords(normalizedBrand)); + + if (words.length < 3 || words.length > 8) { + return false; + } + + if ( + !/[a-zа-яё0-9]/i.test(normalizedValue) || + /\{\{[^}]+}}/.test(value) || + /https?:|\/\/|\.html\b|href\b|logo\b|логотип\b|beforecviz\b|staticelements\b|txt-файл|txt-файла|иконка\b|^[^а-яёa-z0-9]+$/iu.test(normalizedValue) || + /[0-9]{6,}/.test(normalizedValue) + ) { + return false; + } + + if (normalizedValue === normalizedBrand) { + return false; + } + + const meaningfulWords = words.filter((word) => !RUSSIAN_STOP_WORDS.has(word) && !brandWords.has(word) && word.length > 1); + + return meaningfulWords.length >= 2; +} + +function cleanSeedPhrase(value: string) { + return value + .replace(/^\.\.\.\s*/, "") + .replace(/\s*\.\.\.$/, "") + .replace(/[“”«»"]/g, "") + .replace(/[|;:…]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function getWindowedPhrase(sentence: string, term: string) { + const words = cleanSeedPhrase(sentence).split(/\s+/).filter(Boolean); + const normalizedTermWords = getSeedPhraseWords(term); + const normalizedWords = words.map((word) => normalizeSearchText(word)); + const termIndex = normalizedWords.findIndex((word, index) => + normalizedTermWords.every((termWord, termWordIndex) => normalizedWords[index + termWordIndex] === termWord) + ); + + if (termIndex < 0) { + return null; + } + + const start = Math.max(0, termIndex - 2); + const end = Math.min(words.length, termIndex + Math.max(normalizedTermWords.length, 1) + 4); + + return cleanSeedPhrase(words.slice(start, end).join(" ")); +} + +function extractEvidenceSeedPhrases( + cluster: ReturnType[number], + brandName: string +) { + const terms = [...cluster.matchedTerms, ...cluster.queryExamples]; + const candidates: string[] = []; + + for (const snippet of cluster.evidenceSnippets) { + const sentenceCandidates = snippet.text + .split(/[.!?\n\r]+/u) + .map(cleanSeedPhrase) + .filter(Boolean); + + for (const sentence of sentenceCandidates) { + const sentenceHasTerm = terms.some((term) => countTermHits(normalizeSearchText(sentence), term) > 0); + + if (!sentenceHasTerm) { + continue; + } + + if (isSemanticBasisPhrase(sentence, brandName)) { + candidates.push(sentence); + continue; + } + + for (const term of terms) { + if (countTermHits(normalizeSearchText(sentence), term) === 0) { + continue; + } + + const phrase = getWindowedPhrase(sentence, term); + + if (phrase && isSemanticBasisPhrase(phrase, brandName)) { + candidates.push(phrase); + } + } + } + } + + return candidates; +} + +function buildClusterSeedPhrases(cluster: ReturnType[number], brandName: string) { + return Array.from( + new Set( + cluster.queryExamples + .map(cleanSeedPhrase) + .filter((phrase) => isSemanticBasisPhrase(phrase, brandName)) + .map((phrase) => phrase.toLocaleLowerCase("ru-RU")) + ) + ); +} + function getLandingOpportunityPageType( pagePath: string ): SemanticAnalysisOutput["landingOpportunities"][number]["pageType"] { @@ -2523,6 +2797,7 @@ async function buildAnalysisOutput(projectId: string): Promise<{ ontology: Proje const ontology = await getProjectOntology(projectId, ontologyEvidence); const pageScope = buildPageScopeSummary(pageDocuments, contentDocuments); const styleProfile = buildStyleProfile(coverageDocuments.length > 0 ? coverageDocuments : documents, ontology); + const textCorpus = buildTextCorpus(coverageDocuments.length > 0 ? coverageDocuments : documents); const clusters = analyzeClusters(coverageDocuments, ontology); const coverageScore = getCoverageScore(clusters); const lexicalCoverageScore = getLexicalCoverageScore(clusters); @@ -2640,6 +2915,7 @@ async function buildAnalysisOutput(projectId: string): Promise<{ ontology: Proje wordCount: document.wordCount, matchedClusters: contentClusterMap.get(document.id) ?? [] })), + textCorpus, legacyFindings, gaps, landingOpportunities, diff --git a/seo_mode/seo_mode/server/src/business/seoBusinessSynthesis.ts b/seo_mode/seo_mode/server/src/business/seoBusinessSynthesis.ts new file mode 100644 index 0000000..2ea38cd --- /dev/null +++ b/seo_mode/seo_mode/server/src/business/seoBusinessSynthesis.ts @@ -0,0 +1,1047 @@ +import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js"; +import { getLatestAlignedSeoContextReview, type SeoContextReviewRun } from "../contextReview/seoContextReview.js"; +import { pool } from "../db/client.js"; + +type RunStatus = "queued" | "running" | "done" | "failed" | "cancelled"; +type ProviderMode = "codex_manual" | "codex_workspace" | "deterministic_fallback"; +type Grounding = "explicit" | "inferred" | "hypothesis"; + +type SeoBusinessSynthesisRunRow = { + id: string; + status: RunStatus; + input: Record; + output: SeoBusinessSynthesisContract | null; + error_message: string | null; + started_at: Date | null; + completed_at: Date | null; + created_at: Date; +}; + +type BusinessEvidenceLine = { + documentId: string; + id: string; + scope: SemanticAnalysisRun["textCorpus"]["chunks"][number]["scope"]; + selected: boolean; + sourcePath: string; + text: string; + title: string; + urlPath: string | null; + usedForCoverage: boolean; +}; + +export type SeoBusinessSynthesisModelTaskContract = { + taskType: "seo.business_synthesis"; + schemaVersion: "seo-business-synthesis-task.v1"; + taskId: string; + projectId: string; + semanticRunId: string; + projectOntologyVersionId: string | null; + providerMode: "model_provider_contract"; + input: { + siteSummary: { + brandName: string; + language: SemanticAnalysisRun["styleProfile"]["language"]; + topTerms: string[]; + ctaPhrases: string[]; + }; + siteTextEvidence: { + schemaVersion: "seo-business-text-evidence.v1"; + extraction: SemanticAnalysisRun["textCorpus"]["extraction"]; + documents: Array<{ + id: string; + selected: boolean; + sourcePath: string; + title: string; + urlPath: string | null; + usedForCoverage: boolean; + wordCount: number; + }>; + highSignalLines: BusinessEvidenceLine[]; + }; + contextReview: { + status: "available" | "missing"; + summary: string | null; + forbiddenClaims: string[]; + normalizationHints: Array<{ + title: string; + targetIntent: string; + sourcePhrases: string[]; + marketAngles: string[]; + }>; + }; + semanticClusters: Array<{ + id: string; + title: string; + priority: "high" | "medium" | "low"; + status: "covered" | "weak" | "missing"; + targetIntent: string; + matchedTerms: string[]; + queryExamples: string[]; + evidenceSnippets: string[]; + }>; + }; + allowedActions: Array< + | "read_business_context" + | "separate_core_offer_from_application_vectors" + | "infer_conservative_product_hierarchy" + | "propose_demand_families" + | "flag_risky_expansion" + >; + outputSchema: { + schemaVersion: "seo-business-synthesis.v1"; + requiredTopLevelKeys: string[]; + }; + stopConditions: string[]; +}; + +export type SeoBusinessSynthesisContract = { + schemaVersion: "seo-business-synthesis.v1"; + projectId: string; + semanticRunId: string; + projectOntologyVersionId: string | null; + generatedAt: string; + provider: { + mode: ProviderMode; + modelRequired: true; + message: string; + }; + sourceTaskId: string; + analystReview: { + status: "approved" | "needs_changes" | "not_reviewed" | "rejected"; + reviewer: "codex" | "human" | "system"; + reviewedAt: string | null; + notes: string[]; + }; + productFrame: { + coreOffer: string; + productCategory: string; + centralThesis: string; + confidence: "low" | "medium" | "high"; + evidenceRefs: string[]; + }; + semanticHierarchy: { + corePillars: Array<{ + id: string; + title: string; + role: "core_platform" | "platform_layer" | "workflow_layer" | "governance_layer" | "supporting_capability"; + grounding: Grounding; + whyItMatters: string; + evidenceRefs: string[]; + }>; + applicationVectors: Array<{ + id: string; + title: string; + relationshipToCore: "application_of_core_platform" | "explicit_module" | "possible_vertical" | "risky_expansion"; + grounding: Grounding; + priority: "high" | "medium" | "low"; + queryDirection: string; + evidenceRefs: string[]; + }>; + }; + demandFamilies: Array<{ + id: string; + title: string; + role: "core" | "supporting" | "use_case" | "vertical" | "commercial"; + grounding: Grounding; + priority: "high" | "medium" | "low"; + phrases: string[]; + rationale: string; + }>; + guardrails: string[]; + risks: Array<{ + id: string; + title: string; + message: string; + }>; + summary: string; + nextActions: string[]; +}; + +export type SeoBusinessSynthesisRun = SeoBusinessSynthesisContract & { + runId: string; + status: RunStatus; + input: Record; + startedAt: string | null; + completedAt: string | null; + errorMessage: string | null; + createdAt: string; +}; + +type SeoBusinessSynthesisModelProviderFallback = { + modelTask: SeoBusinessSynthesisModelTaskContract; + projectOntologyVersionId: string | null; + semanticRunId: string; +}; + +function toIsoDate(value: Date | null) { + return value ? value.toISOString() : null; +} + +function getDefaultAnalystReview(): SeoBusinessSynthesisContract["analystReview"] { + return { + notes: [], + reviewedAt: null, + reviewer: "system", + status: "not_reviewed" + }; +} + +function mapSeoBusinessSynthesisRun(row: SeoBusinessSynthesisRunRow): SeoBusinessSynthesisRun | null { + if (!row.output) { + return null; + } + + return { + runId: row.id, + status: row.status, + input: row.input, + startedAt: toIsoDate(row.started_at), + completedAt: toIsoDate(row.completed_at), + errorMessage: row.error_message, + createdAt: row.created_at.toISOString(), + ...row.output, + analystReview: row.output.analystReview ?? getDefaultAnalystReview() + }; +} + +function asRecord(value: unknown): Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)) ? value as Record : {}; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asString(value: unknown) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function asBusinessLabel(value: unknown): string | null { + const direct = asString(value); + + if (direct) { + return direct; + } + + const record = asRecord(value); + + return ( + asString(record.label) ?? + asString(record.title) ?? + asString(record.name) ?? + asString(record.summary) ?? + asString(record.description) ?? + asString(record.intent) ?? + null + ); +} + +function asStringArray(value: unknown): string[] { + if (typeof value === "string" && value.trim().length > 0) { + return [value.trim()]; + } + + return asArray(value).map(asString).filter((item): item is string => Boolean(item)); +} + +function asBusinessStringArray(value: unknown): string[] { + const direct = asStringArray(value); + + if (direct.length > 0) { + return direct; + } + + return asArray(value).map(asBusinessLabel).filter((item): item is string => Boolean(item)); +} + +function pickBusinessArray(record: Record, keys: string[]): string[] { + return keys.flatMap((key) => asBusinessStringArray(record[key])); +} + +function pickEvidence(record: Record) { + return pickBusinessArray(record, [ + "evidenceRefs", + "evidence", + "evidenceBasis", + "sourceRefs", + "sourcePhrases", + "sources" + ]).slice(0, 12); +} + +function getHierarchyLevelNodes(rawHierarchy: Record, predicate: (level: Record) => boolean) { + return asArray(rawHierarchy.levels).flatMap((level) => { + const record = asRecord(level); + + return predicate(record) ? asArray(record.nodes) : []; + }); +} + +function normalizeText(value: string) { + return value + .trim() + .replace(/[“”«»"]/g, "") + .replace(/[|;:…]+/g, " ") + .replace(/\s+/g, " ") + .toLocaleLowerCase("ru-RU"); +} + +function toId(value: string) { + return normalizeText(value) + .replace(/[^a-zа-яё0-9]+/gi, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); +} + +function unique(values: string[]) { + const seen = new Set(); + const result: string[] = []; + + for (const value of values) { + const normalizedValue = normalizeText(value); + + if (!normalizedValue || seen.has(normalizedValue)) { + continue; + } + + seen.add(normalizedValue); + result.push(normalizedValue); + } + + return result; +} + +function getWords(value: string) { + return normalizeText(value).match(/[a-zа-яё0-9-]{3,}/giu) ?? []; +} + +function splitEvidenceLines(text: string) { + return text + .split(/\n+|[•●▪]/u) + .flatMap((line) => line.split(/(?<=[.!?])\s+/u)) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); +} + +function isBusinessEvidenceLine(value: string) { + const words = getWords(value); + + if (words.length < 4 || words.length > 22) { + return false; + } + + return ( + /[а-яёa-z]/i.test(value) && + value.length <= 220 && + !/\{\{[^}]+}}|<[^>]+>|\b(class|src|href|data-[\w-]+)\s*=|https?:\/\/|\.(png|jpe?g|webp|svg|gif|css|js)\b/i.test(value) && + !/\b(content|template|beforecviz|staticelements)\.[a-z0-9_.-]+|html\s+html|media\s+attr|boolean\s+boolean|txt-файл/iu.test(value) + ); +} + +function getBusinessEvidenceLines(analysis: SemanticAnalysisRun) { + const seen = new Set(); + const rows: Array = []; + + for (const chunk of analysis.textCorpus.chunks) { + if (!chunk.selected && !chunk.usedForCoverage && chunk.scope !== "indexable_page" && chunk.scope !== "content_source") { + continue; + } + + splitEvidenceLines(chunk.text).forEach((line, index) => { + const key = normalizeText(line); + + if (!isBusinessEvidenceLine(line) || seen.has(key)) { + return; + } + + seen.add(key); + rows.push({ + documentId: chunk.documentId, + id: `${chunk.id}:business-line:${index + 1}`, + scope: chunk.scope, + selected: chunk.selected, + sourcePath: chunk.sourcePath, + text: line, + title: chunk.title, + urlPath: chunk.urlPath, + usedForCoverage: chunk.usedForCoverage, + score: + (chunk.selected ? 40 : 0) + + (chunk.usedForCoverage ? 50 : 0) + + (chunk.scope === "indexable_page" ? 25 : chunk.scope === "content_source" ? 15 : 0) + + Math.min(24, getWords(line).length) + }); + }); + } + + return rows + .sort((left, right) => right.score - left.score || left.sourcePath.localeCompare(right.sourcePath)) + .slice(0, 140) + .map(({ score: _score, ...line }) => line); +} + +export function buildSeoBusinessSynthesisTask( + projectId: string, + analysis: SemanticAnalysisRun, + contextReview: SeoContextReviewRun | null = null +): SeoBusinessSynthesisModelTaskContract { + return { + taskType: "seo.business_synthesis", + schemaVersion: "seo-business-synthesis-task.v1", + taskId: `${analysis.runId}:seo-business-synthesis:v1`, + projectId, + semanticRunId: analysis.runId, + projectOntologyVersionId: analysis.projectOntology.versionId, + providerMode: "model_provider_contract", + input: { + siteSummary: { + brandName: analysis.projectOntology.brandName, + language: analysis.styleProfile.language, + topTerms: analysis.styleProfile.topTerms.slice(0, 30).map((term) => term.term), + ctaPhrases: analysis.styleProfile.ctaPhrases.slice(0, 12) + }, + siteTextEvidence: { + schemaVersion: "seo-business-text-evidence.v1", + extraction: analysis.textCorpus.extraction, + documents: analysis.textCorpus.documents.slice(0, 40).map((document) => ({ + id: document.id, + selected: document.selected, + sourcePath: document.sourcePath, + title: document.title, + urlPath: document.urlPath, + usedForCoverage: document.usedForCoverage, + wordCount: document.wordCount + })), + highSignalLines: getBusinessEvidenceLines(analysis) + }, + contextReview: { + status: contextReview ? "available" : "missing", + summary: contextReview?.summary ?? null, + forbiddenClaims: contextReview?.forbiddenClaims.map((claim) => claim.claim).slice(0, 20) ?? [], + normalizationHints: + contextReview?.normalizationHints.slice(0, 30).map((hint) => ({ + marketAngles: hint.marketAngles.slice(0, 8), + sourcePhrases: hint.sourcePhrases.slice(0, 10), + targetIntent: hint.targetIntent, + title: hint.title + })) ?? [] + }, + semanticClusters: analysis.clusters.slice(0, 24).map((cluster) => ({ + id: cluster.id, + priority: cluster.priority, + status: cluster.status, + title: cluster.title, + targetIntent: cluster.targetIntent, + matchedTerms: cluster.matchedTerms.slice(0, 16), + queryExamples: cluster.queryExamples.slice(0, 10), + evidenceSnippets: cluster.evidenceSnippets.slice(0, 6).map((snippet) => snippet.text) + })) + }, + allowedActions: [ + "read_business_context", + "separate_core_offer_from_application_vectors", + "infer_conservative_product_hierarchy", + "propose_demand_families", + "flag_risky_expansion" + ], + outputSchema: { + schemaVersion: "seo-business-synthesis.v1", + requiredTopLevelKeys: [ + "productFrame", + "semanticHierarchy", + "demandFamilies" + ] + }, + stopConditions: [ + "Do not turn application vectors into the core product category.", + "Keep the central offer as the highest-confidence reading of the selected site text.", + "Separate explicit site claims from inferred hypotheses and risky expansion.", + "Demand families are proposals for human review, not Wordstat/SERP approval.", + "If a vertical is only an example of the platform, mark it as application_of_core_platform or possible_vertical." + ] + }; +} + +function normalizeGrounding(value: unknown, fallback: Grounding = "inferred"): Grounding { + if (value === "explicit" || value === "inferred" || value === "hypothesis") { + return value; + } + + const text = normalizeText(asString(value) ?? ""); + + if (!text) { + return fallback; + } + + if (/(risky|risk|speculative|unsupported|hypothesis|гипотез)/iu.test(text)) { + return "hypothesis"; + } + + if (/(infer|inferred|inference|conservative|caveat|possible|вывод|предполож)/iu.test(text)) { + return "inferred"; + } + + if (/(explicit|source|evidence|direct|прям|явн|доказ)/iu.test(text)) { + return "explicit"; + } + + return fallback; +} + +function normalizePriority(value: unknown): "high" | "medium" | "low" { + if (value === "high" || value === "medium" || value === "low") { + return value; + } + + const text = normalizeText(asString(value) ?? ""); + + if (/(highest|critical|highest-priority|высок|главн|core)/iu.test(text)) { + return "high"; + } + + if (/(low|weak|optional|низк)/iu.test(text)) { + return "low"; + } + + return "medium"; +} + +function normalizeCorePillarRole( + record: Record, + index: number +): SeoBusinessSynthesisContract["semanticHierarchy"]["corePillars"][number]["role"] { + const exactRole = asString(record.role); + + if ( + exactRole === "core_platform" || + exactRole === "platform_layer" || + exactRole === "workflow_layer" || + exactRole === "governance_layer" || + exactRole === "supporting_capability" + ) { + return exactRole; + } + + const text = normalizeText([ + exactRole, + asString(record.type), + asString(record.classification), + asString(record.relationToCore), + asString(record.id), + asBusinessLabel(record.label), + asBusinessLabel(record.title), + asBusinessLabel(record.name) + ].filter(Boolean).join(" ")); + + if (/(semantic-center|semantic_center|central|center|центр|core-offer|core_offer)/iu.test(text)) { + return "core_platform"; + } + + if (/(workflow|scenario|automation|integration|api|процесс|сценари|интеграц|автоматизац)/iu.test(text)) { + return "workflow_layer"; + } + + if (/(access|role|permission|identity|security|governance|admin|доступ|роль|прав|безопас|администр)/iu.test(text)) { + return "governance_layer"; + } + + if (/(layer|core-layer|core_layer|core-platform-layer|core_platform_layer|cross-cutting|cross_cutting|platform|платформ|контур|слой)/iu.test(text)) { + return "platform_layer"; + } + + return index === 0 ? "core_platform" : "supporting_capability"; +} + +function normalizeApplicationVectorRelationship( + record: Record +): SeoBusinessSynthesisContract["semanticHierarchy"]["applicationVectors"][number]["relationshipToCore"] { + const exactRelationship = asString(record.relationshipToCore); + + if ( + exactRelationship === "application_of_core_platform" || + exactRelationship === "explicit_module" || + exactRelationship === "possible_vertical" || + exactRelationship === "risky_expansion" + ) { + return exactRelationship; + } + + const text = normalizeText([ + exactRelationship, + asString(record.type), + asString(record.classification), + asString(record.relationToCore), + asString(record.status), + asString(record.claimType), + asBusinessLabel(record.label), + asBusinessLabel(record.title) + ].filter(Boolean).join(" ")); + + if (/(risky|risk|unsupported|опас|риск)/iu.test(text)) { + return "risky_expansion"; + } + + if (/(vertical|industry|segment|отрасл|вертикал)/iu.test(text)) { + return "possible_vertical"; + } + + if (/(module|application|explicit-module|explicit_module|модул|приложен)/iu.test(text)) { + return "explicit_module"; + } + + return "application_of_core_platform"; +} + +function normalizeDemandFamilyRole(record: Record): SeoBusinessSynthesisContract["demandFamilies"][number]["role"] { + const exactRole = asString(record.role); + + if (exactRole === "core" || exactRole === "supporting" || exactRole === "use_case" || exactRole === "vertical" || exactRole === "commercial") { + return exactRole; + } + + const text = normalizeText([ + exactRole, + asString(record.id), + asString(record.type), + asString(record.relationToCore), + asString(record.intent), + asString(record.status), + asBusinessLabel(record.title), + asBusinessLabel(record.label) + ].filter(Boolean).join(" ")); + + if (/(core-offer|core_offer|core-platform|core_platform|semantic-center|semantic_center|главн|основн|ядро)/iu.test(text)) { + return "core"; + } + + if (/(^|\s)(commercial|price|demo|tariff|buy)(\s|$)|стоимост|(^|\s)(цена|цены|ценам|ценой)(\s|$)|тариф|демо/iu.test(text)) { + return "commercial"; + } + + if (/(vertical|industry|отрасл|вертикал)/iu.test(text)) { + return "vertical"; + } + + if (/(use-case|use_case|scenario|module|сценари|модул|кейс)/iu.test(text)) { + return "use_case"; + } + + return "supporting"; +} + +function normalizeBusinessSynthesisShape( + projectId: string, + output: SeoBusinessSynthesisContract, + fallback: SeoBusinessSynthesisModelProviderFallback +): SeoBusinessSynthesisContract { + const rawOutput = asRecord(output); + const rawProductFrame = asRecord(rawOutput.productFrame); + const rawHierarchy = asRecord(rawOutput.semanticHierarchy); + const centralOffer = asRecord(rawProductFrame.centralOffer); + const positioning = asRecord(rawProductFrame.positioning); + const productCategory = + asString(rawProductFrame.productCategory) ?? + asBusinessLabel(rawProductFrame.category) ?? + asBusinessLabel(rawProductFrame.productCategory) ?? + asBusinessLabel(rawProductFrame.centralOffer) ?? + asBusinessLabel(rawProductFrame.offer) ?? + fallback.modelTask.input.siteSummary.brandName; + const coreOffer = + asString(rawProductFrame.coreOffer) ?? + asBusinessLabel(rawProductFrame.centralOffer) ?? + asBusinessLabel(rawProductFrame.offer) ?? + asBusinessLabel(rawProductFrame.primaryValueProposition) ?? + productCategory; + const productFrame = { + confidence: + rawProductFrame.confidence === "high" || rawProductFrame.confidence === "medium" || rawProductFrame.confidence === "low" + ? rawProductFrame.confidence + : centralOffer.confidence === "high" || centralOffer.confidence === "medium" || centralOffer.confidence === "low" + ? centralOffer.confidence + : "medium", + coreOffer, + productCategory, + centralThesis: + asString(rawProductFrame.centralThesis) ?? + asString(rawProductFrame.summary) ?? + asString(rawProductFrame.description) ?? + asString(centralOffer.description) ?? + asString(centralOffer.summary) ?? + asBusinessLabel(rawProductFrame.positioning) ?? + asString(positioning.description) ?? + coreOffer, + evidenceRefs: [ + ...asStringArray(rawProductFrame.evidenceRefs), + ...pickEvidence(centralOffer), + ...pickEvidence(positioning), + ...pickEvidence(asRecord(rawProductFrame.primaryValueProposition)) + ].slice(0, 16) + } satisfies SeoBusinessSynthesisContract["productFrame"]; + const rawCorePillarItems = [ + ...asArray(rawHierarchy.corePillars), + ...asArray(rawProductFrame.corePillars) + ]; + const corePillarItems = rawCorePillarItems.length > 0 + ? rawCorePillarItems + : getHierarchyLevelNodes(rawHierarchy, (level) => { + const text = normalizeText([ + asString(level.name), + asString(level.type), + asString(level.level) + ].filter(Boolean).join(" ")); + + return /\b(core|platform|layer|слой|платформ)\b/iu.test(text); + }); + const corePillars: SeoBusinessSynthesisContract["semanticHierarchy"]["corePillars"] = corePillarItems.map((item, index) => { + const record = asRecord(item); + const title = + asString(record.title) ?? + asBusinessLabel(record.label) ?? + asBusinessLabel(record.name) ?? + asBusinessLabel(record.heading) ?? + `Платформенный смысл ${index + 1}`; + + return { + evidenceRefs: pickEvidence(record), + grounding: normalizeGrounding(record.grounding ?? record.claimType, "explicit"), + id: asString(record.id) ?? `pillar.${index + 1}`, + role: normalizeCorePillarRole(record, index), + title, + whyItMatters: + asString(record.whyItMatters) ?? + asString(record.meaning) ?? + asString(record.reason) ?? + asString(record.rationale) ?? + asString(record.description) ?? + asString(record.claimType) ?? + "Смысл найден в контексте сайта." + }; + }); + const rawApplicationVectorItems = [ + ...asArray(rawHierarchy.applicationVectors), + ...asArray(rawProductFrame.applicationVectors) + ]; + const applicationVectorItems = rawApplicationVectorItems.length > 0 + ? rawApplicationVectorItems + : getHierarchyLevelNodes(rawHierarchy, (level) => { + const text = normalizeText([ + asString(level.name), + asString(level.type), + asString(level.level) + ].filter(Boolean).join(" ")); + + return /\b(application|vector|vertical|use-case|use_case|приклад|сценари|вертикал)\b/iu.test(text); + }); + const applicationVectors: SeoBusinessSynthesisContract["semanticHierarchy"]["applicationVectors"] = applicationVectorItems.map((item, index) => { + const record = asRecord(item); + const title = + asString(record.title) ?? + asBusinessLabel(record.label) ?? + asBusinessLabel(record.name) ?? + `Сценарий применения ${index + 1}`; + + return { + evidenceRefs: pickEvidence(record), + grounding: normalizeGrounding(record.grounding ?? record.claimType ?? record.status ?? record.type), + id: asString(record.id) ?? `vector.${index + 1}`, + priority: normalizePriority(record.priority ?? record.confidence), + queryDirection: + asString(record.queryDirection) ?? + asString(record.demandDirection) ?? + asString(record.seoDirection) ?? + asString(record.intent) ?? + asString(record.description) ?? + "Проверить как дочерний сценарий применения, не как ядро продукта.", + relationshipToCore: normalizeApplicationVectorRelationship(record), + title + }; + }); + const demandFamilies: SeoBusinessSynthesisContract["demandFamilies"] = asArray(rawOutput.demandFamilies).map((item, index) => { + const record = asRecord(item); + const title = + asString(record.title) ?? + asBusinessLabel(record.label) ?? + asBusinessLabel(record.name) ?? + `Семейство спроса ${index + 1}`; + const phrases = unique([ + ...asStringArray(record.phrases), + ...asStringArray(record.corePhrases), + ...asStringArray(record.queryExamples), + ...asStringArray(record.seedPhrases), + ...asStringArray(record.wordstatQueries), + title + ]).slice(0, 12); + + return { + grounding: normalizeGrounding(record.grounding ?? record.claimType ?? record.status), + id: asString(record.id) ?? `demand.${index + 1}`, + phrases, + priority: normalizePriority(record.priority ?? record.confidence), + rationale: + asString(record.rationale) ?? + asString(record.reason) ?? + asString(record.intent) ?? + asString(record.humanReviewNotes) ?? + "Demand family proposed from business synthesis.", + role: normalizeDemandFamilyRole(record), + title + }; + }); + const fallbackDemandFamilies = + demandFamilies.length > 0 + ? demandFamilies + : [ + { + grounding: "inferred" as const, + id: "demand.core", + phrases: unique([ + productFrame.productCategory, + productFrame.coreOffer, + ...corePillars.slice(0, 4).map((pillar) => pillar.title) + ]).slice(0, 8), + priority: "high" as const, + rationale: "Fallback family from product frame and core pillars.", + role: "core" as const, + title: productFrame.productCategory + } + ]; + + return { + ...output, + analystReview: output.analystReview ?? getDefaultAnalystReview(), + demandFamilies: fallbackDemandFamilies, + generatedAt: asString(rawOutput.generatedAt) ?? new Date().toISOString(), + guardrails: + asStringArray(rawOutput.guardrails).length > 0 + ? asStringArray(rawOutput.guardrails) + : [ + "Не превращать дочерние vertical/application vectors в core product category.", + "Все demand families остаются proposal до ручной фиксации semantic basis." + ], + nextActions: + asStringArray(rawOutput.nextActions).length > 0 + ? asStringArray(rawOutput.nextActions) + : ["Передать business synthesis в seo.normalization как смысловую рамку перед Stage 05."], + productFrame, + projectId: asString(rawOutput.projectId) ?? projectId, + projectOntologyVersionId: + asString(rawOutput.projectOntologyVersionId) ?? fallback.projectOntologyVersionId, + provider: { + message: asString(asRecord(rawOutput.provider).message) ?? "Business synthesis normalized by backend.", + mode: "codex_workspace", + modelRequired: true + }, + risks: asArray(rawOutput.risks).map((risk, index) => { + const record = asRecord(risk); + + return { + id: asString(record.id) ?? `risk.${index + 1}`, + message: asString(record.message) ?? asString(record.reason) ?? "Проверить риск перед нормализацией спроса.", + title: asString(record.title) ?? `Риск ${index + 1}` + }; + }), + schemaVersion: "seo-business-synthesis.v1", + semanticHierarchy: { + applicationVectors, + corePillars: + corePillars.length > 0 + ? corePillars + : [ + { + evidenceRefs: [], + grounding: "inferred", + id: "pillar.core", + role: "core_platform", + title: productFrame.productCategory, + whyItMatters: productFrame.centralThesis + } + ] + }, + semanticRunId: asString(rawOutput.semanticRunId) ?? fallback.semanticRunId, + sourceTaskId: asString(rawOutput.sourceTaskId) ?? fallback.modelTask.taskId, + summary: asString(rawOutput.summary) ?? productFrame.centralThesis + }; +} + +function normalizeBusinessSynthesisOutput( + projectId: string, + output: SeoBusinessSynthesisContract, + providerMode: ProviderMode, + providerMessage: string, + fallback: SeoBusinessSynthesisModelProviderFallback +) { + const normalizedOutput = normalizeBusinessSynthesisShape(projectId, output, fallback); + + if (normalizedOutput.schemaVersion !== "seo-business-synthesis.v1") { + throw new Error("Business synthesis output schemaVersion должен быть seo-business-synthesis.v1."); + } + + if (normalizedOutput.projectId !== projectId || !normalizedOutput.semanticRunId || !normalizedOutput.sourceTaskId) { + throw new Error("Business synthesis output должен иметь projectId, semanticRunId и sourceTaskId."); + } + + if (!normalizedOutput.productFrame.coreOffer || !normalizedOutput.productFrame.productCategory) { + throw new Error("Business synthesis output должен иметь productFrame.coreOffer и productFrame.productCategory."); + } + + return { + ...normalizedOutput, + generatedAt: new Date().toISOString(), + provider: { + message: providerMessage, + mode: providerMode, + modelRequired: true + } + }; +} + +function buildFallbackBusinessSynthesis(projectId: string, analysis: SemanticAnalysisRun): SeoBusinessSynthesisContract { + const task = buildSeoBusinessSynthesisTask(projectId, analysis); + const coreClusters = analysis.clusters.filter((cluster) => cluster.priority === "high").slice(0, 6); + const vectors: SeoBusinessSynthesisContract["semanticHierarchy"]["applicationVectors"] = coreClusters.slice(1, 8).map((cluster) => ({ + evidenceRefs: cluster.sourceRefs.slice(0, 3).map((ref) => `${cluster.id}:${ref.id}`), + grounding: cluster.evidence.level === "none" ? "hypothesis" as const : "explicit" as const, + id: cluster.id, + priority: cluster.priority, + queryDirection: cluster.targetIntent, + relationshipToCore: "application_of_core_platform" as const, + title: cluster.title + })); + const coreOffer = analysis.analysisVerdict.summary; + + return { + schemaVersion: "seo-business-synthesis.v1", + projectId, + semanticRunId: analysis.runId, + projectOntologyVersionId: analysis.projectOntology.versionId, + generatedAt: new Date().toISOString(), + provider: { + mode: "deterministic_fallback", + modelRequired: true, + message: "Business synthesis fallback собран из semantic clusters; model pass должен заменить его продуктовым reading." + }, + sourceTaskId: task.taskId, + analystReview: getDefaultAnalystReview(), + productFrame: { + confidence: analysis.analysisVerdict.confidence, + coreOffer, + productCategory: coreClusters[0]?.title ?? analysis.projectOntology.brandName, + centralThesis: analysis.analysisVerdict.summary, + evidenceRefs: coreClusters[0]?.sourceRefs.slice(0, 4).map((ref) => `${coreClusters[0].id}:${ref.id}`) ?? [] + }, + semanticHierarchy: { + corePillars: coreClusters.slice(0, 5).map((cluster, index) => ({ + evidenceRefs: cluster.sourceRefs.slice(0, 3).map((ref) => `${cluster.id}:${ref.id}`), + grounding: cluster.evidence.level === "none" ? "hypothesis" : "explicit", + id: cluster.id, + role: index === 0 ? "core_platform" : "supporting_capability", + title: cluster.title, + whyItMatters: cluster.targetIntent + })), + applicationVectors: vectors + }, + demandFamilies: coreClusters.slice(0, 10).map((cluster, index) => ({ + grounding: cluster.evidence.level === "none" ? "hypothesis" : "inferred", + id: `demand.${cluster.id}`, + phrases: unique([cluster.title, ...cluster.queryExamples, ...cluster.matchedTerms]).slice(0, 8), + priority: cluster.priority, + rationale: index === 0 ? "Fallback treats the strongest high-priority cluster as product center." : cluster.targetIntent, + role: index === 0 ? "core" : "use_case", + title: cluster.title + })), + guardrails: [ + "Fallback не должен сам расширять рынок; model pass обязан отделить core offer от application vectors.", + "Дочерние сценарии остаются proposal до ручной фиксации semantic basis." + ], + risks: analysis.analysisVerdict.risks.slice(0, 10).map((risk) => ({ + id: risk.id, + message: risk.message, + title: risk.title + })), + summary: analysis.analysisVerdict.summary, + nextActions: ["Передать business synthesis в seo.normalization как смысловую рамку."] + }; +} + +export async function getLatestAlignedSeoBusinessSynthesis( + projectId: string, + semanticRunId: string +): Promise { + const result = await pool.query( + ` + select id, status, input, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 + and run_type = 'seo_business_synthesis' + and output->>'semanticRunId' = $2 + order by created_at desc + limit 1; + `, + [projectId, semanticRunId] + ); + + return result.rows[0] ? mapSeoBusinessSynthesisRun(result.rows[0]) : null; +} + +export async function getSeoBusinessSynthesisContract(projectId: string): Promise { + const analysis = await getLatestSemanticAnalysis(projectId); + + if (!analysis) { + return null; + } + + const aligned = await getLatestAlignedSeoBusinessSynthesis(projectId, analysis.runId); + + return aligned ?? buildFallbackBusinessSynthesis(projectId, analysis); +} + +export async function saveSeoBusinessSynthesisFromModelProvider( + projectId: string, + output: SeoBusinessSynthesisContract, + sourceModelTaskRunId: string, + fallback: SeoBusinessSynthesisModelProviderFallback +) { + const normalizedOutput = normalizeBusinessSynthesisOutput( + projectId, + output, + "codex_workspace", + "AI Workspace Codex provider выполнил seo.business_synthesis по contract-only task; результат сохранён как product-frame evidence.", + fallback + ); + const result = await pool.query( + ` + insert into runs (project_id, run_type, status, input, output, started_at, completed_at) + values ($1, 'seo_business_synthesis', 'done', $2::jsonb, $3::jsonb, now(), now()) + returning id, status, input, output, error_message, started_at, completed_at, created_at; + `, + [ + projectId, + JSON.stringify({ + providerMode: "codex_workspace", + schemaVersion: "seo-business-synthesis-model-provider-input.v1", + semanticRunId: normalizedOutput.semanticRunId, + sourceModelTaskRunId, + sourceTaskId: normalizedOutput.sourceTaskId, + taskType: "seo.business_synthesis" + }), + JSON.stringify(normalizedOutput) + ] + ); + const run = result.rows[0] ? mapSeoBusinessSynthesisRun(result.rows[0]) : null; + + if (!run) { + throw new Error("Не удалось сохранить AI Workspace Business synthesis."); + } + + return run; +} + +export function buildSeoBusinessSynthesisPromotionFallback( + modelTask: SeoBusinessSynthesisModelTaskContract +): SeoBusinessSynthesisModelProviderFallback { + return { + modelTask, + projectOntologyVersionId: modelTask.projectOntologyVersionId, + semanticRunId: modelTask.semanticRunId + }; +} diff --git a/seo_mode/seo_mode/server/src/commercial/seoCommercialDemand.ts b/seo_mode/seo_mode/server/src/commercial/seoCommercialDemand.ts new file mode 100644 index 0000000..7b20846 --- /dev/null +++ b/seo_mode/seo_mode/server/src/commercial/seoCommercialDemand.ts @@ -0,0 +1,732 @@ +import { getLatestSemanticAnalysis } from "../analysis/semanticAnalysis.js"; +import { + getLatestAlignedSeoBusinessSynthesis, + getSeoBusinessSynthesisContract, + type SeoBusinessSynthesisContract +} from "../business/seoBusinessSynthesis.js"; +import { pool } from "../db/client.js"; + +type RunStatus = "queued" | "running" | "done" | "failed" | "cancelled"; +type ProviderMode = "codex_manual" | "codex_workspace" | "deterministic_fallback"; +type Grounding = "explicit" | "inferred" | "hypothesis"; +type BuyerIntent = + | "solution_search" + | "vendor_selection" + | "implementation" + | "automation" + | "integration" + | "replacement" + | "pilot" + | "commercial"; + +type SeoCommercialDemandRunRow = { + id: string; + status: RunStatus; + input: Record; + output: SeoCommercialDemandContract | null; + error_message: string | null; + started_at: Date | null; + completed_at: Date | null; + created_at: Date; +}; + +export type SeoCommercialDemandModelTaskContract = { + taskType: "seo.commercial_demand"; + schemaVersion: "seo-commercial-demand-task.v1"; + taskId: string; + projectId: string; + semanticRunId: string; + projectOntologyVersionId: string | null; + providerMode: "model_provider_contract"; + input: { + siteSummary: { + brandName: string; + language: string; + topTerms: string[]; + }; + businessSynthesis: { + productFrame: SeoBusinessSynthesisContract["productFrame"]; + corePillars: SeoBusinessSynthesisContract["semanticHierarchy"]["corePillars"]; + applicationVectors: SeoBusinessSynthesisContract["semanticHierarchy"]["applicationVectors"]; + demandFamilies: SeoBusinessSynthesisContract["demandFamilies"]; + guardrails: string[]; + summary: string; + }; + }; + allowedActions: Array< + | "identify_buyer_context" + | "convert_architecture_to_commercial_value" + | "preserve_core_differentiator" + | "separate_commercial_queries_from_informational_queries" + | "flag_noncommercial_or_overbroad_angles" + >; + outputSchema: { + schemaVersion: "seo-commercial-demand.v1"; + requiredTopLevelKeys: string[]; + }; + stopConditions: string[]; +}; + +export type SeoCommercialDemandContract = { + schemaVersion: "seo-commercial-demand.v1"; + projectId: string; + semanticRunId: string; + projectOntologyVersionId: string | null; + generatedAt: string; + provider: { + mode: ProviderMode; + modelRequired: true; + message: string; + }; + sourceTaskId: string; + analystReview: { + status: "approved" | "needs_changes" | "not_reviewed" | "rejected"; + reviewer: "codex" | "human" | "system"; + reviewedAt: string | null; + notes: string[]; + }; + commercialCore: { + coreBuyerProblem: string; + coreCommercialValue: string; + coreDifferentiator: string; + buyingTrigger: string; + decisionContext: string; + confidence: "low" | "medium" | "high"; + evidenceRefs: string[]; + }; + buyerIntentFamilies: Array<{ + id: string; + title: string; + buyerIntent: BuyerIntent; + priority: "high" | "medium" | "low"; + grounding: Grounding; + commercialAngle: string; + sourceMeaning: string; + coreQualifier: string; + queryPatterns: string[]; + excludedInformationalPatterns: string[]; + evidenceRefs: string[]; + }>; + applicationCommercialAngles: Array<{ + id: string; + sourceVector: string; + relationshipToCore: string; + priority: "high" | "medium" | "low"; + grounding: Grounding; + commercialReframe: string; + coreQualifier: string; + queryPatterns: string[]; + evidenceRefs: string[]; + }>; + rejectedAngles: Array<{ + id: string; + title: string; + reason: string; + suggestedCommercialReframe: string | null; + }>; + guardrails: string[]; + summary: string; + nextActions: string[]; +}; + +export type SeoCommercialDemandRun = SeoCommercialDemandContract & { + runId: string; + status: RunStatus; + input: Record; + startedAt: string | null; + completedAt: string | null; + errorMessage: string | null; + createdAt: string; +}; + +type SeoCommercialDemandModelProviderFallback = { + modelTask: SeoCommercialDemandModelTaskContract; + projectOntologyVersionId: string | null; + semanticRunId: string; +}; + +function toIsoDate(value: Date | null) { + return value ? value.toISOString() : null; +} + +function asRecord(value: unknown): Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)) ? value as Record : {}; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asString(value: unknown) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function asLabel(value: unknown): string | null { + const direct = asString(value); + + if (direct) { + return direct; + } + + const record = asRecord(value); + + return asString(record.title) ?? asString(record.label) ?? asString(record.name) ?? asString(record.summary) ?? asString(record.description); +} + +function asStringArray(value: unknown): string[] { + if (typeof value === "string" && value.trim().length > 0) { + return [value.trim()]; + } + + return asArray(value).map(asLabel).filter((item): item is string => Boolean(item)); +} + +function normalizeText(value: string) { + return value + .trim() + .replace(/[“”«»"]/g, "") + .replace(/[|,;:…]+/g, " ") + .replace(/[\\/]+/g, " ") + .replace(/\s+/g, " ") + .toLocaleLowerCase("ru-RU"); +} + +function unique(values: string[]) { + const seen = new Set(); + const result: string[] = []; + + for (const value of values) { + const normalizedValue = normalizeText(value); + + if (!normalizedValue || seen.has(normalizedValue)) { + continue; + } + + seen.add(normalizedValue); + result.push(normalizedValue); + } + + return result; +} + +function toId(value: string) { + return normalizeText(value) + .replace(/[^a-zа-яё0-9]+/gi, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); +} + +function normalizeGrounding(value: unknown, fallback: Grounding = "inferred"): Grounding { + if (value === "explicit" || value === "inferred" || value === "hypothesis") { + return value; + } + + const text = normalizeText(asString(value) ?? ""); + + if (/(risk|unsupported|hypothesis|гипотез|не подтверж)/iu.test(text)) { + return "hypothesis"; + } + + if (/(infer|possible|conservative|предполож|вывод)/iu.test(text)) { + return "inferred"; + } + + if (/(explicit|source|evidence|direct|явн|доказ)/iu.test(text)) { + return "explicit"; + } + + return fallback; +} + +function normalizePriority(value: unknown): "high" | "medium" | "low" { + if (value === "high" || value === "medium" || value === "low") { + return value; + } + + const text = normalizeText(asString(value) ?? ""); + + if (/(highest|critical|high|core|главн|высок)/iu.test(text)) { + return "high"; + } + + if (/(low|weak|optional|низк)/iu.test(text)) { + return "low"; + } + + return "medium"; +} + +function normalizeBuyerIntent(value: unknown, text: string): BuyerIntent { + const normalizedValue = normalizeText(`${asString(value) ?? ""} ${text}`); + + if (/(pilot|demo|trial|пилот|демо|заявк|подключ)/iu.test(normalizedValue)) { + return "pilot"; + } + + if (/(implement|deploy|rollout|внедрен|запуск|разверт)/iu.test(normalizedValue)) { + return "implementation"; + } + + if (/(integrat|api|webhook|1c|1с|интеграц|подключ)/iu.test(normalizedValue)) { + return "integration"; + } + + if (/(replace|alternative|вместо|замен|альтернатив)/iu.test(normalizedValue)) { + return "replacement"; + } + + if (/(vendor|platform|service|provider|поставщик|подрядчик|платформ|сервис|решение)/iu.test(normalizedValue)) { + return "vendor_selection"; + } + + if (/(automat|agent|workflow|автоматизац|агент|процесс)/iu.test(normalizedValue)) { + return "automation"; + } + + if (/(commercial|price|tariff|цена|стоим|тариф)/iu.test(normalizedValue)) { + return "commercial"; + } + + return "solution_search"; +} + +function getDefaultAnalystReview(): SeoCommercialDemandContract["analystReview"] { + return { + notes: [], + reviewedAt: null, + reviewer: "system", + status: "not_reviewed" + }; +} + +function mapSeoCommercialDemandRun(row: SeoCommercialDemandRunRow): SeoCommercialDemandRun | null { + if (!row.output) { + return null; + } + + return { + runId: row.id, + status: row.status, + input: row.input, + startedAt: toIsoDate(row.started_at), + completedAt: toIsoDate(row.completed_at), + errorMessage: row.error_message, + createdAt: row.created_at.toISOString(), + ...row.output, + analystReview: row.output.analystReview ?? getDefaultAnalystReview() + }; +} + +export function buildSeoCommercialDemandTask( + projectId: string, + businessSynthesis: SeoBusinessSynthesisContract +): SeoCommercialDemandModelTaskContract { + return { + taskType: "seo.commercial_demand", + schemaVersion: "seo-commercial-demand-task.v1", + taskId: `${businessSynthesis.semanticRunId}:seo-commercial-demand:v1`, + projectId, + semanticRunId: businessSynthesis.semanticRunId, + projectOntologyVersionId: businessSynthesis.projectOntologyVersionId, + providerMode: "model_provider_contract", + input: { + siteSummary: { + brandName: businessSynthesis.productFrame.coreOffer, + language: "mixed", + topTerms: [] + }, + businessSynthesis: { + productFrame: businessSynthesis.productFrame, + corePillars: businessSynthesis.semanticHierarchy.corePillars.slice(0, 12), + applicationVectors: businessSynthesis.semanticHierarchy.applicationVectors.slice(0, 18), + demandFamilies: businessSynthesis.demandFamilies.slice(0, 20), + guardrails: businessSynthesis.guardrails.slice(0, 20), + summary: businessSynthesis.summary + } + }, + allowedActions: [ + "identify_buyer_context", + "convert_architecture_to_commercial_value", + "preserve_core_differentiator", + "separate_commercial_queries_from_informational_queries", + "flag_noncommercial_or_overbroad_angles" + ], + outputSchema: { + schemaVersion: "seo-commercial-demand.v1", + requiredTopLevelKeys: [ + "commercialCore", + "buyerIntentFamilies", + "applicationCommercialAngles" + ] + }, + stopConditions: [ + "Do not output architecture inventory as keyword demand.", + "Every query pattern must express buyer intent, business value, implementation intent, vendor selection, automation, replacement, integration, pilot, or commercial evaluation.", + "Preserve the site's core differentiator from productFrame/corePillars; do not detach verticals into unrelated markets.", + "If an application vector only makes sense with the core platform differentiator, include that qualifier in query patterns.", + "Do not call Wordstat/SERP or claim market demand is proven." + ] + }; +} + +function getEvidenceRefs(record: Record) { + return unique([ + ...asStringArray(record.evidenceRefs), + ...asStringArray(record.evidence), + ...asStringArray(record.evidenceBasis), + ...asStringArray(record.sources) + ]).slice(0, 12); +} + +function normalizeCommercialDemandShape( + projectId: string, + output: SeoCommercialDemandContract, + fallback: SeoCommercialDemandModelProviderFallback +): SeoCommercialDemandContract { + const rawOutput = asRecord(output); + const rawCore = asRecord(rawOutput.commercialCore); + const fallbackProductFrame = fallback.modelTask.input.businessSynthesis.productFrame; + const coreDifferentiator = + asString(rawCore.coreDifferentiator) ?? + asString(rawCore.differentiator) ?? + fallbackProductFrame.coreOffer; + const commercialCore: SeoCommercialDemandContract["commercialCore"] = { + buyingTrigger: + asString(rawCore.buyingTrigger) ?? + asString(rawCore.trigger) ?? + "Покупатель ищет решение, когда текущие процессы, данные, роли или исполнение требуют системного улучшения.", + confidence: + rawCore.confidence === "high" || rawCore.confidence === "medium" || rawCore.confidence === "low" + ? rawCore.confidence + : fallbackProductFrame.confidence, + coreBuyerProblem: + asString(rawCore.coreBuyerProblem) ?? + asString(rawCore.buyerProblem) ?? + asString(rawCore.problem) ?? + fallbackProductFrame.centralThesis, + coreCommercialValue: + asString(rawCore.coreCommercialValue) ?? + asString(rawCore.commercialValue) ?? + asString(rawCore.value) ?? + fallbackProductFrame.coreOffer, + coreDifferentiator, + decisionContext: + asString(rawCore.decisionContext) ?? + asString(rawCore.context) ?? + "Выбор, внедрение или пилот решения внутри бизнес/операционного контура.", + evidenceRefs: getEvidenceRefs(rawCore) + }; + const buyerIntentFamilies = asArray(rawOutput.buyerIntentFamilies).map((item, index) => { + const record = asRecord(item); + const title = asString(record.title) ?? asLabel(record.name) ?? `Коммерческий intent ${index + 1}`; + const queryPatterns = unique([ + ...asStringArray(record.queryPatterns), + ...asStringArray(record.queries), + ...asStringArray(record.corePhrases), + ...asStringArray(record.phrases) + ]).slice(0, 12); + + return { + buyerIntent: normalizeBuyerIntent(record.buyerIntent ?? record.intent, `${title} ${queryPatterns.join(" ")}`), + commercialAngle: + asString(record.commercialAngle) ?? + asString(record.angle) ?? + asString(record.rationale) ?? + "Покупатель оценивает практическую бизнес-ценность решения.", + coreQualifier: asString(record.coreQualifier) ?? asString(record.qualifier) ?? coreDifferentiator, + evidenceRefs: getEvidenceRefs(record), + excludedInformationalPatterns: unique([ + ...asStringArray(record.excludedInformationalPatterns), + ...asStringArray(record.excludedPatterns), + ...asStringArray(record.nonCommercialPatterns) + ]).slice(0, 10), + grounding: normalizeGrounding(record.grounding ?? record.claimType, "inferred"), + id: asString(record.id) ?? `buyer.${toId(title) || index + 1}`, + priority: normalizePriority(record.priority), + queryPatterns, + sourceMeaning: asString(record.sourceMeaning) ?? asString(record.meaning) ?? title, + title + }; + }); + const applicationCommercialAngles = asArray(rawOutput.applicationCommercialAngles).map((item, index) => { + const record = asRecord(item); + const sourceVector = asString(record.sourceVector) ?? asString(record.title) ?? asLabel(record.name) ?? `Application vector ${index + 1}`; + + return { + commercialReframe: + asString(record.commercialReframe) ?? + asString(record.reframe) ?? + asString(record.commercialAngle) ?? + "Переформулировать как покупательский сценарий, связанный с ядром продукта.", + coreQualifier: asString(record.coreQualifier) ?? asString(record.qualifier) ?? coreDifferentiator, + evidenceRefs: getEvidenceRefs(record), + grounding: normalizeGrounding(record.grounding ?? record.claimType, "inferred"), + id: asString(record.id) ?? `angle.${toId(sourceVector) || index + 1}`, + priority: normalizePriority(record.priority), + queryPatterns: unique([ + ...asStringArray(record.queryPatterns), + ...asStringArray(record.queries), + ...asStringArray(record.phrases) + ]).slice(0, 10), + relationshipToCore: asString(record.relationshipToCore) ?? asString(record.relationship) ?? "application_of_core_platform", + sourceVector + }; + }); + const fallbackBuyerIntentFamilies = + buyerIntentFamilies.length > 0 + ? buyerIntentFamilies + : fallback.modelTask.input.businessSynthesis.demandFamilies.slice(0, 10).map((family, index) => ({ + buyerIntent: normalizeBuyerIntent(family.role, family.title), + commercialAngle: family.rationale, + coreQualifier: coreDifferentiator, + evidenceRefs: [], + excludedInformationalPatterns: [], + grounding: family.grounding, + id: `buyer.${family.id || index + 1}`, + priority: family.priority, + queryPatterns: family.phrases.slice(0, 8), + sourceMeaning: family.title, + title: family.title + })); + + return { + ...output, + analystReview: output.analystReview ?? getDefaultAnalystReview(), + applicationCommercialAngles, + buyerIntentFamilies: fallbackBuyerIntentFamilies, + commercialCore, + generatedAt: asString(rawOutput.generatedAt) ?? new Date().toISOString(), + guardrails: + asStringArray(rawOutput.guardrails).length > 0 + ? asStringArray(rawOutput.guardrails) + : [ + "Не превращать архитектурный тезис в запрос без покупательской ценности.", + "Прикладной сценарий должен оставаться связанным с core differentiator продукта.", + "Wordstat/SERP запускаются только после ручной фиксации semantic basis." + ], + nextActions: + asStringArray(rawOutput.nextActions).length > 0 + ? asStringArray(rawOutput.nextActions) + : ["Передать commercial demand framing в seo.normalization для формирования human-gated basis."], + projectId: asString(rawOutput.projectId) ?? projectId, + projectOntologyVersionId: + asString(rawOutput.projectOntologyVersionId) ?? fallback.projectOntologyVersionId, + provider: { + message: asString(asRecord(rawOutput.provider).message) ?? "Commercial demand normalized by backend.", + mode: "codex_workspace", + modelRequired: true + }, + rejectedAngles: asArray(rawOutput.rejectedAngles).map((item, index) => { + const record = asRecord(item); + + return { + id: asString(record.id) ?? `rejected.${index + 1}`, + reason: asString(record.reason) ?? "Недостаточно коммерческого buyer intent.", + suggestedCommercialReframe: asString(record.suggestedCommercialReframe) ?? asString(record.suggestedReplacement), + title: asString(record.title) ?? asString(record.phrase) ?? `Некоммерческий угол ${index + 1}` + }; + }), + schemaVersion: "seo-commercial-demand.v1", + semanticRunId: asString(rawOutput.semanticRunId) ?? fallback.semanticRunId, + sourceTaskId: asString(rawOutput.sourceTaskId) ?? fallback.modelTask.taskId, + summary: + asString(rawOutput.summary) ?? + `${commercialCore.coreBuyerProblem} ${commercialCore.coreCommercialValue}`.trim() + }; +} + +function normalizeCommercialDemandOutput( + projectId: string, + output: SeoCommercialDemandContract, + providerMode: ProviderMode, + providerMessage: string, + fallback: SeoCommercialDemandModelProviderFallback +) { + const normalizedOutput = normalizeCommercialDemandShape(projectId, output, fallback); + + if (normalizedOutput.schemaVersion !== "seo-commercial-demand.v1") { + throw new Error("Commercial demand output schemaVersion должен быть seo-commercial-demand.v1."); + } + + if (normalizedOutput.projectId !== projectId || !normalizedOutput.semanticRunId || !normalizedOutput.sourceTaskId) { + throw new Error("Commercial demand output должен иметь projectId, semanticRunId и sourceTaskId."); + } + + if (!normalizedOutput.commercialCore.coreBuyerProblem || !normalizedOutput.commercialCore.coreDifferentiator) { + throw new Error("Commercial demand output должен иметь commercialCore.coreBuyerProblem и coreDifferentiator."); + } + + return { + ...normalizedOutput, + generatedAt: new Date().toISOString(), + provider: { + message: providerMessage, + mode: providerMode, + modelRequired: true + } + }; +} + +function buildFallbackCommercialDemand( + projectId: string, + businessSynthesis: SeoBusinessSynthesisContract +): SeoCommercialDemandContract { + const task = buildSeoCommercialDemandTask(projectId, businessSynthesis); + const coreDifferentiator = businessSynthesis.productFrame.coreOffer; + + return { + schemaVersion: "seo-commercial-demand.v1", + projectId, + semanticRunId: businessSynthesis.semanticRunId, + projectOntologyVersionId: businessSynthesis.projectOntologyVersionId, + generatedAt: new Date().toISOString(), + provider: { + mode: "deterministic_fallback", + modelRequired: true, + message: "Commercial demand fallback собран из business synthesis; model pass должен заменить архитектурные тезисы buyer-intent формулировками." + }, + sourceTaskId: task.taskId, + analystReview: getDefaultAnalystReview(), + commercialCore: { + buyingTrigger: "Покупатель ищет практическое решение вокруг выявленного ядра продукта.", + confidence: businessSynthesis.productFrame.confidence, + coreBuyerProblem: businessSynthesis.productFrame.centralThesis, + coreCommercialValue: businessSynthesis.productFrame.coreOffer, + coreDifferentiator, + decisionContext: "Выбор, внедрение или пилот решения.", + evidenceRefs: businessSynthesis.productFrame.evidenceRefs.slice(0, 8) + }, + buyerIntentFamilies: businessSynthesis.demandFamilies.slice(0, 10).map((family, index) => ({ + buyerIntent: normalizeBuyerIntent(family.role, family.title), + commercialAngle: family.rationale, + coreQualifier: coreDifferentiator, + evidenceRefs: [], + excludedInformationalPatterns: [], + grounding: family.grounding, + id: `buyer.${family.id || index + 1}`, + priority: family.priority, + queryPatterns: family.phrases.slice(0, 8), + sourceMeaning: family.title, + title: family.title + })), + applicationCommercialAngles: businessSynthesis.semanticHierarchy.applicationVectors.slice(0, 10).map((vector, index) => ({ + commercialReframe: vector.queryDirection, + coreQualifier: coreDifferentiator, + evidenceRefs: vector.evidenceRefs.slice(0, 8), + grounding: vector.grounding, + id: `angle.${vector.id || index + 1}`, + priority: vector.priority, + queryPatterns: [], + relationshipToCore: vector.relationshipToCore, + sourceVector: vector.title + })), + rejectedAngles: [], + guardrails: [ + "Fallback не доказывает рынок и не запускает Wordstat/SERP.", + "Model pass должен заменить архитектурные тезисы покупательскими формулировками." + ], + summary: businessSynthesis.summary, + nextActions: ["Запустить seo.commercial_demand через model provider перед normalization."] + }; +} + +export async function getLatestAlignedSeoCommercialDemand( + projectId: string, + semanticRunId: string +): Promise { + const result = await pool.query( + ` + select id, status, input, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 + and run_type = 'seo_commercial_demand' + and output->>'semanticRunId' = $2 + order by created_at desc + limit 1; + `, + [projectId, semanticRunId] + ); + + return result.rows[0] ? mapSeoCommercialDemandRun(result.rows[0]) : null; +} + +export async function getSeoCommercialDemandContract(projectId: string): Promise { + const analysis = await getLatestSemanticAnalysis(projectId); + + if (!analysis) { + return null; + } + + const aligned = await getLatestAlignedSeoCommercialDemand(projectId, analysis.runId); + + if (aligned) { + return aligned; + } + + const businessSynthesis = await getSeoBusinessSynthesisContract(projectId); + + return businessSynthesis ? buildFallbackCommercialDemand(projectId, businessSynthesis) : null; +} + +export async function buildLatestSeoCommercialDemandTask(projectId: string): Promise { + const analysis = await getLatestSemanticAnalysis(projectId); + + if (!analysis) { + return null; + } + + const businessSynthesis = await getLatestAlignedSeoBusinessSynthesis(projectId, analysis.runId); + + return businessSynthesis ? buildSeoCommercialDemandTask(projectId, businessSynthesis) : null; +} + +export async function saveSeoCommercialDemandFromModelProvider( + projectId: string, + output: SeoCommercialDemandContract, + sourceModelTaskRunId: string, + fallback: SeoCommercialDemandModelProviderFallback +) { + const normalizedOutput = normalizeCommercialDemandOutput( + projectId, + output, + "codex_workspace", + "AI Workspace Codex provider выполнил seo.commercial_demand; результат сохранён как buyer-intent evidence.", + fallback + ); + const result = await pool.query( + ` + insert into runs (project_id, run_type, status, input, output, started_at, completed_at) + values ($1, 'seo_commercial_demand', 'done', $2::jsonb, $3::jsonb, now(), now()) + returning id, status, input, output, error_message, started_at, completed_at, created_at; + `, + [ + projectId, + JSON.stringify({ + providerMode: "codex_workspace", + schemaVersion: "seo-commercial-demand-model-provider-input.v1", + semanticRunId: normalizedOutput.semanticRunId, + sourceModelTaskRunId, + sourceTaskId: normalizedOutput.sourceTaskId, + taskType: "seo.commercial_demand" + }), + JSON.stringify(normalizedOutput) + ] + ); + const run = result.rows[0] ? mapSeoCommercialDemandRun(result.rows[0]) : null; + + if (!run) { + throw new Error("Не удалось сохранить AI Workspace Commercial demand."); + } + + return run; +} + +export function buildSeoCommercialDemandPromotionFallback( + modelTask: SeoCommercialDemandModelTaskContract +): SeoCommercialDemandModelProviderFallback { + return { + modelTask, + projectOntologyVersionId: modelTask.projectOntologyVersionId, + semanticRunId: modelTask.semanticRunId + }; +} diff --git a/seo_mode/seo_mode/server/src/config/env.ts b/seo_mode/seo_mode/server/src/config/env.ts index 1b2f8e8..334bf05 100644 --- a/seo_mode/seo_mode/server/src/config/env.ts +++ b/seo_mode/seo_mode/server/src/config/env.ts @@ -25,7 +25,7 @@ const envSchema = z.object({ WORDSTAT_REGION_IDS: z.string().default("225"), WORDSTAT_DEVICES: z.string().default("DEVICE_ALL"), WORDSTAT_NUM_PHRASES: z.coerce.number().int().positive().default(50), - WORDSTAT_MAX_SEEDS_PER_RUN: z.coerce.number().int().positive().max(80).default(40), + WORDSTAT_MAX_SEEDS_PER_RUN: z.coerce.number().int().positive().max(80).default(12), SERP_PROVIDER: z.enum(["disabled", "external_api"]).default("disabled"), SERP_API_BASE_URL: z.string().url().optional(), SERP_API_TOKEN: z.string().optional(), diff --git a/seo_mode/seo_mode/server/src/contextReview/seoContextReview.ts b/seo_mode/seo_mode/server/src/contextReview/seoContextReview.ts index e3770dc..1be76bf 100644 --- a/seo_mode/seo_mode/server/src/contextReview/seoContextReview.ts +++ b/seo_mode/seo_mode/server/src/contextReview/seoContextReview.ts @@ -17,7 +17,14 @@ type SeoContextReviewRunRow = { type ReviewLevel = "ok" | "warning" | "problem"; type ContextReviewConfidence = "low" | "medium" | "high"; -type SeoContextReviewProviderMode = "codex_manual" | "deterministic_fallback"; +type SeoContextReviewProviderMode = "codex_manual" | "codex_workspace" | "deterministic_fallback"; + +type SeoContextReviewModelProviderFallback = { + projectOntologyVersionId: string | null; + scanVersionId: string; + semanticRunId: string; + sourceTaskId: string; +}; export type SeoContextReviewOutput = { schemaVersion: "seo-context-review.v1"; @@ -166,52 +173,316 @@ function getDefaultAnalystReview(): SeoContextReviewOutput["analystReview"] { }; } +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function asStringArray(value: unknown): string[] { + return asArray(value).map(asString).filter((item): item is string => Boolean(item)); +} + +function asBoolean(value: unknown, fallback = false) { + return typeof value === "boolean" ? value : fallback; +} + +function normalizeConfidence(value: unknown, fallback: ContextReviewConfidence = "medium"): ContextReviewConfidence { + const normalized = asString(value)?.toLowerCase(); + + if (normalized === "high" || normalized === "medium" || normalized === "low") { + return normalized; + } + + return normalized === "moderate" ? "medium" : fallback; +} + +function normalizeReviewLevel(value: unknown): ReviewLevel { + const normalized = asString(value)?.toLowerCase(); + + return normalized === "problem" || normalized === "warning" || normalized === "ok" ? normalized : "warning"; +} + +function normalizeEvidenceRefs(value: unknown): string[] { + return asArray(value) + .map((item, index) => { + if (typeof item === "string") { + return item.trim(); + } + + const record = asRecord(item); + const sourcePath = asString(record.sourcePath) ?? asString(record.ref) ?? asString(record.id); + const reason = asString(record.reason) ?? asString(record.title); + + return [sourcePath, reason].filter(Boolean).join(" · ") || `evidence:${index + 1}`; + }) + .filter(Boolean) + .slice(0, 24); +} + +function normalizeContextReviewShape( + projectId: string, + output: SeoContextReviewOutput, + fallback?: SeoContextReviewModelProviderFallback +): SeoContextReviewOutput { + const rawOutput = asRecord(output); + const rawSiteIdentity = asRecord(rawOutput.siteIdentity); + const rawOffer = asRecord(rawOutput.offer); + const rawAudience = asRecord(rawOutput.audience); + const summary = asString(rawOutput.summary) ?? "Context Review model output сохранен после backend normalization."; + const siteIdentityDescription = + asString(rawSiteIdentity.description) ?? asString(rawSiteIdentity.identitySummary) ?? summary; + const offerSummary = asString(rawOffer.summary) ?? asString(rawOffer.primaryOffer) ?? summary; + const audiencePrimary = + asString(rawAudience.primary) ?? + asStringArray(rawAudience.primaryAudiences)[0] ?? + asStringArray(rawAudience.decisionMakers)[0] ?? + "Целевая аудитория требует ручной проверки."; + const providerReadiness = asRecord(rawOutput.readiness); + const needsHumanReview = asBoolean( + rawOutput.needsHumanReview, + asBoolean(providerReadiness.needsHumanReview, true) + ); + const confirmedClaims = + asArray(rawOffer.confirmedClaims).length > 0 + ? asArray(rawOffer.confirmedClaims).map((claim, index) => { + const record = asRecord(claim); + + return { + confidence: normalizeConfidence(record.confidence), + evidenceRefs: normalizeEvidenceRefs(record.evidenceRefs), + id: asString(record.id) ?? `claim.${index + 1}`, + title: asString(record.title) ?? asString(record.claim) ?? `Подтвержденный смысл ${index + 1}` + }; + }) + : [ + { + confidence: normalizeConfidence(rawOffer.confidence, "medium"), + evidenceRefs: normalizeEvidenceRefs(rawOffer.evidenceRefs), + id: "claim.primary_offer", + title: offerSummary + }, + ...asStringArray(rawOffer.supportingCapabilities) + .slice(0, 8) + .map((title, index) => ({ + confidence: normalizeConfidence(rawOffer.confidence, "medium"), + evidenceRefs: normalizeEvidenceRefs(rawOffer.evidenceRefs), + id: `claim.capability.${index + 1}`, + title + })) + ]; + const pageRoles = asArray(rawOutput.pageRoles).map((role, index) => { + const record = asRecord(role); + const rawRole = asString(record.role); + const normalizedRole: SeoContextReviewOutput["pageRoles"][number]["role"] = + rawRole === "content_source" || asString(record.scope) === "content_source" + ? "content_source" + : rawRole === "template_block" || asString(record.scope) === "template_block" + ? "template_block" + : rawRole === "admin_or_technical" || rawRole === "non_seo_admin_surface" || asString(record.scope) === "admin_page" + ? "admin_or_technical" + : rawRole === "primary_landing" || rawRole === "primary_product_landing" || index === 0 + ? "primary_landing" + : "supporting_page"; + + return { + evidenceRefs: normalizeEvidenceRefs(record.evidenceRefs), + pageId: asString(record.pageId) ?? asString(record.id) ?? `page.${index + 1}`, + path: asString(record.path) ?? asString(record.urlPath), + reason: asString(record.reason) ?? asString(record.risk) ?? "Model provider page role.", + role: normalizedRole, + scope: asString(record.scope) ?? "unknown", + title: asString(record.title) ?? `Страница ${index + 1}` + }; + }); + const sectionFindings = asArray(rawOutput.sectionFindings).map((finding, index) => { + const record = asRecord(finding); + const status = asString(record.status); + const evidenceLevel = asString(record.evidenceLevel); + const normalizedEvidenceLevel: SeoContextReviewOutput["sectionFindings"][number]["evidenceLevel"] = + evidenceLevel === "none" || evidenceLevel === "thin" || evidenceLevel === "moderate" || evidenceLevel === "strong" + ? evidenceLevel + : normalizeConfidence(record.confidence) === "high" + ? "moderate" + : "thin"; + const normalizedStatus: SeoContextReviewOutput["sectionFindings"][number]["status"] = + status === "covered" || status === "weak" || status === "missing" + ? status + : asBoolean(record.needsHumanReview) + ? "weak" + : "covered"; + + return { + confidence: normalizeConfidence(record.confidence), + evidenceLevel: normalizedEvidenceLevel, + evidenceRefs: normalizeEvidenceRefs(record.evidenceRefs), + id: asString(record.id) ?? `section.${index + 1}`, + status: normalizedStatus, + targetIntent: asString(record.targetIntent) ?? asString(record.risk) ?? "Проверить смысл секции.", + title: asString(record.title) ?? `Секция ${index + 1}` + }; + }); + const semanticGaps = asArray(rawOutput.semanticGaps).map((gap, index) => { + const record = asRecord(gap); + const priority = asString(record.priority); + const normalizedPriority: SeoContextReviewOutput["semanticGaps"][number]["priority"] = + priority === "high" || priority === "medium" || priority === "low" ? priority : "medium"; + + return { + id: asString(record.id) ?? `gap.${index + 1}`, + priority: normalizedPriority, + reason: asString(record.reason) ?? asString(record.description) ?? "Нужна ручная проверка.", + suggestedReview: asString(record.suggestedReview) ?? "Проверить перед отправкой в спрос.", + title: asString(record.title) ?? `Семантический пробел ${index + 1}` + }; + }); + const forbiddenClaims = asArray(rawOutput.forbiddenClaims).map((claim, index) => { + const record = asRecord(claim); + const source = asString(record.source); + const normalizedSource: SeoContextReviewOutput["forbiddenClaims"][number]["source"] = + source === "semantic_gap" || source === "weak_evidence" || source === "guardrail" ? source : "guardrail"; + + return { + claim: typeof claim === "string" ? claim : asString(record.claim) ?? asString(record.title) ?? `Запрет ${index + 1}`, + id: asString(record.id) ?? `forbidden.${index + 1}`, + reason: asString(record.reason) ?? "Guardrail из Context Review.", + source: normalizedSource + }; + }); + const normalizationHints = asArray(rawOutput.normalizationHints).map((hint, index) => { + const record = asRecord(hint); + + return { + clusterId: asString(record.clusterId) ?? asString(record.id) ?? `hint.${index + 1}`, + id: asString(record.id) ?? `hint.${index + 1}`, + marketAngles: asStringArray(record.marketAngles), + reason: asString(record.reason) ?? asString(record.hint) ?? "Normalization hint from model provider.", + reviewRequired: asBoolean(record.reviewRequired, true), + sourcePhrases: asStringArray(record.sourcePhrases), + stopPhrases: asStringArray(record.stopPhrases), + targetIntent: asString(record.targetIntent) ?? asString(record.title) ?? "Проверить нормализацию.", + title: asString(record.title) ?? `Подсказка нормализации ${index + 1}` + }; + }); + const risks = asArray(rawOutput.risks).map((risk, index) => { + const record = asRecord(risk); + + return { + id: asString(record.id) ?? `risk.${index + 1}`, + level: normalizeReviewLevel(record.level), + message: asString(record.message) ?? asString(record.reason) ?? "Risk from model provider.", + title: asString(record.title) ?? `Риск ${index + 1}` + }; + }); + + return { + ...output, + analystReview: output.analystReview ?? getDefaultAnalystReview(), + audience: { + confidence: normalizeConfidence(rawAudience.confidence), + primary: audiencePrimary, + reason: asString(rawAudience.reason) ?? "Audience normalized from model provider output.", + secondary: asStringArray(rawAudience.secondary) + }, + forbiddenClaims, + needsHumanReview, + normalizationHints, + offer: { + confirmedClaims, + summary: offerSummary, + weakClaims: asArray(rawOffer.weakClaims).map((claim, index) => { + const record = asRecord(claim); + + return { + id: asString(record.id) ?? `weak_claim.${index + 1}`, + reason: asString(record.reason) ?? "Нужна проверка доказательности.", + title: asString(record.title) ?? asString(record.claim) ?? `Слабое утверждение ${index + 1}` + }; + }) + }, + pageRoles, + projectId: asString(rawOutput.projectId) ?? projectId, + projectOntologyVersionId: asString(rawOutput.projectOntologyVersionId) ?? fallback?.projectOntologyVersionId ?? null, + readiness: { + evidenceRefCount: Number(providerReadiness.evidenceRefCount) || normalizeEvidenceRefs(rawSiteIdentity.evidenceRefs).length, + forbiddenClaimCount: forbiddenClaims.length, + needsHumanReview, + normalizationHintCount: normalizationHints.length, + pageRoleCount: pageRoles.length, + semanticGapCount: semanticGaps.length + }, + risks, + scanVersionId: asString(rawOutput.scanVersionId) ?? fallback?.scanVersionId ?? "", + schemaVersion: "seo-context-review.v1", + sectionFindings, + semanticGaps, + semanticRunId: asString(rawOutput.semanticRunId) ?? fallback?.semanticRunId ?? "", + siteIdentity: { + brandName: asString(rawSiteIdentity.brandName) ?? "Проект", + confidence: normalizeConfidence(rawSiteIdentity.confidence), + description: siteIdentityDescription, + evidenceRefs: normalizeEvidenceRefs(rawSiteIdentity.evidenceRefs) + }, + sourceTaskId: asString(rawOutput.sourceTaskId) ?? fallback?.sourceTaskId ?? "", + summary + }; +} + function normalizeContextReviewOutput( projectId: string, output: SeoContextReviewOutput, providerMode: SeoContextReviewProviderMode, - providerMessage: string + providerMessage: string, + fallback?: SeoContextReviewModelProviderFallback ): SeoContextReviewOutput { - if (output.schemaVersion !== "seo-context-review.v1") { + const normalizedShape = normalizeContextReviewShape(projectId, output, fallback); + + if (normalizedShape.schemaVersion !== "seo-context-review.v1") { throw new Error("Context Review output schemaVersion должен быть seo-context-review.v1."); } - if (output.projectId !== projectId) { + if (normalizedShape.projectId !== projectId) { throw new Error("Context Review output projectId не совпадает с проектом."); } - if (!output.semanticRunId || !output.scanVersionId || !output.sourceTaskId) { + if (!normalizedShape.semanticRunId || !normalizedShape.scanVersionId || !normalizedShape.sourceTaskId) { throw new Error("Context Review output должен иметь semanticRunId, scanVersionId и sourceTaskId."); } - if (!output.siteIdentity?.description || !output.offer?.summary || !output.summary) { + if (!normalizedShape.siteIdentity?.description || !normalizedShape.offer?.summary || !normalizedShape.summary) { throw new Error("Context Review output должен иметь siteIdentity.description, offer.summary и summary."); } - if (!Array.isArray(output.pageRoles) || !Array.isArray(output.sectionFindings)) { + if (!Array.isArray(normalizedShape.pageRoles) || !Array.isArray(normalizedShape.sectionFindings)) { throw new Error("Context Review output должен иметь pageRoles и sectionFindings."); } - if (!Array.isArray(output.semanticGaps) || !Array.isArray(output.forbiddenClaims) || !Array.isArray(output.normalizationHints)) { + if (!Array.isArray(normalizedShape.semanticGaps) || !Array.isArray(normalizedShape.forbiddenClaims) || !Array.isArray(normalizedShape.normalizationHints)) { throw new Error("Context Review output должен иметь semanticGaps, forbiddenClaims и normalizationHints."); } return { - ...output, + ...normalizedShape, generatedAt: new Date().toISOString(), provider: { message: providerMessage, mode: providerMode, modelRequired: true }, - analystReview: output.analystReview ?? getDefaultAnalystReview(), + analystReview: normalizedShape.analystReview ?? getDefaultAnalystReview(), readiness: { - ...output.readiness, - forbiddenClaimCount: output.forbiddenClaims.length, - normalizationHintCount: output.normalizationHints.length, - pageRoleCount: output.pageRoles.length, - semanticGapCount: output.semanticGaps.length, - needsHumanReview: output.needsHumanReview + ...normalizedShape.readiness, + forbiddenClaimCount: normalizedShape.forbiddenClaims.length, + normalizationHintCount: normalizedShape.normalizationHints.length, + pageRoleCount: normalizedShape.pageRoles.length, + semanticGapCount: normalizedShape.semanticGaps.length, + needsHumanReview: normalizedShape.needsHumanReview } }; } @@ -541,3 +812,45 @@ export async function saveSeoContextReviewManual( return run; } + +export async function saveSeoContextReviewFromModelProvider( + projectId: string, + output: SeoContextReviewOutput, + sourceModelTaskRunId: string, + fallback: SeoContextReviewModelProviderFallback +): Promise { + const normalizedOutput = normalizeContextReviewOutput( + projectId, + output, + "codex_workspace", + "AI Workspace Codex provider вернул seo.context_review task как structured JSON; backend schema validation passed.", + fallback + ); + const result = await pool.query( + ` + insert into runs (project_id, run_type, status, input, output, started_at, completed_at) + values ($1, 'seo_context_review', 'done', $2::jsonb, $3::jsonb, now(), now()) + returning id, status, input, output, error_message, started_at, completed_at, created_at; + `, + [ + projectId, + JSON.stringify({ + providerMode: "codex_workspace", + scanVersionId: normalizedOutput.scanVersionId, + schemaVersion: "seo-context-review-model-provider-input.v1", + semanticRunId: normalizedOutput.semanticRunId, + sourceModelTaskRunId, + sourceTaskId: normalizedOutput.sourceTaskId, + taskType: "seo.context_review" + }), + JSON.stringify(normalizedOutput) + ] + ); + const run = result.rows[0] ? mapSeoContextReviewRun(result.rows[0]) : null; + + if (!run) { + throw new Error("Не удалось сохранить AI Workspace Context Review."); + } + + return run; +} diff --git a/seo_mode/seo_mode/server/src/keywords/anchorReview.ts b/seo_mode/seo_mode/server/src/keywords/anchorReview.ts index 23ad1ba..39840e3 100644 --- a/seo_mode/seo_mode/server/src/keywords/anchorReview.ts +++ b/seo_mode/seo_mode/server/src/keywords/anchorReview.ts @@ -1,9 +1,12 @@ import { pool } from "../db/client.js"; import { getSeoNormalizationContract, type SeoNormalizationContract } from "../normalization/seoNormalization.js"; -type AnchorDecisionStatus = "approved" | "disabled" | "pending"; +type AnchorDecisionStatus = "approved" | "deleted" | "disabled" | "pending"; type AnchorDecisionSource = "human" | "model" | "system"; type AnchorReviewState = "approved" | "empty" | "needs_review" | "not_ready"; +export type DemandCollectionProfile = "contextual" | "market_wide"; + +export const DEFAULT_DEMAND_COLLECTION_PROFILE: DemandCollectionProfile = "contextual"; type SeoNormalizationSeed = SeoNormalizationContract["seedStrategy"][number]; export type AnchorReviewDecisionInput = { @@ -80,6 +83,7 @@ export type AnchorReviewContract = { projectOntologyVersionId: string | null; generatedAt: string; state: AnchorReviewState; + demandCollectionProfile: DemandCollectionProfile; source: { normalizationGeneratedAt: string | null; normalizationProviderMode: SeoNormalizationContract["provider"]["mode"] | null; @@ -104,6 +108,13 @@ export type AnchorReviewContract = { manualDecisionCount: number; }; anchors: AnchorReviewAnchor[]; + deletedAnchors: Array<{ + deletedAt: string | null; + id: string; + key: string; + phrase: string; + reason: string; + }>; wordstatQueue: AnchorReviewWordstatQueueItem[]; nextActions: string[]; }; @@ -124,6 +135,10 @@ function normalizePhrase(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } +export function normalizeDemandCollectionProfile(value: unknown): DemandCollectionProfile { + return value === "market_wide" ? "market_wide" : DEFAULT_DEMAND_COLLECTION_PROFILE; +} + function getAnchorKey(seed: Pick) { return `${seed.clusterId}:${normalizePhrase(seed.phrase)}`; } @@ -153,6 +168,18 @@ function getDefaultDecision(seed: SeoNormalizationSeed): AnchorReviewDecision { } function normalizeSavedDecision(seed: SeoNormalizationSeed, decision: AnchorReviewDecision): AnchorReviewDecision { + if (decision.status === "deleted") { + return { + decidedAt: decision.decidedAt, + decidedBy: decision.decidedBy, + reason: decision.reason || "Anchor deleted by review.", + sendToSerp: false, + sendToWordstat: false, + source: decision.source, + status: "deleted" + }; + } + if (decision.status === "approved") { return { decidedAt: decision.decidedAt, @@ -189,15 +216,47 @@ function normalizeSavedDecision(seed: SeoNormalizationSeed, decision: AnchorRevi } function getSavedDecisionSnapshots(savedOutput: AnchorReviewContract | null): DecisionSnapshot[] { - return (savedOutput?.anchors ?? []).map((anchor) => ({ + const anchorSnapshots = (savedOutput?.anchors ?? []).map((anchor) => ({ anchor, decision: anchor.decision, id: anchor.id, key: anchor.key })); + const deletedSnapshots = + savedOutput?.deletedAnchors?.map((anchor) => ({ + decision: { + decidedAt: anchor.deletedAt, + decidedBy: "dev-mode", + reason: anchor.reason || "Anchor deleted by review.", + sendToSerp: false, + sendToWordstat: false, + source: "human" as const, + status: "deleted" as const + }, + id: anchor.id, + key: anchor.key + })) ?? []; + + return [...anchorSnapshots, ...deletedSnapshots]; +} + +function getSavedDemandCollectionProfile(savedOutput: AnchorReviewContract | null): DemandCollectionProfile { + return normalizeDemandCollectionProfile(savedOutput?.demandCollectionProfile); } function normalizeSavedManualDecision(decision: AnchorReviewDecision): AnchorReviewDecision { + if (decision.status === "deleted") { + return { + decidedAt: decision.decidedAt, + decidedBy: decision.decidedBy, + reason: decision.reason || "Manual anchor deleted by review.", + sendToSerp: false, + sendToWordstat: false, + source: decision.source, + status: "deleted" + }; + } + if (decision.status === "approved") { return { decidedAt: decision.decidedAt, @@ -312,6 +371,16 @@ function buildWordstatQueue(anchors: AnchorReviewAnchor[]): AnchorReviewWordstat })); } +function mapDeletedAnchor(anchor: AnchorReviewAnchor) { + return { + deletedAt: anchor.decision.decidedAt, + id: anchor.id, + key: anchor.key, + phrase: anchor.phrase, + reason: anchor.decision.reason || "Anchor deleted by review." + }; +} + function getState(normalization: SeoNormalizationContract, anchors: AnchorReviewAnchor[], wordstatQueueCount: number): AnchorReviewState { if (normalization.state !== "ready" || !normalization.semanticRunId) { return "not_ready"; @@ -324,6 +393,18 @@ function getState(normalization: SeoNormalizationContract, anchors: AnchorReview return wordstatQueueCount > 0 ? "approved" : "needs_review"; } +function getStateFromContract(contract: AnchorReviewContract, anchors: AnchorReviewAnchor[], wordstatQueueCount: number): AnchorReviewState { + if (contract.source.normalizationState !== "ready" || !contract.semanticRunId) { + return "not_ready"; + } + + if (anchors.length === 0) { + return "empty"; + } + + return wordstatQueueCount > 0 ? "approved" : "needs_review"; +} + function buildNextActions(contract: Omit) { const actions: string[] = []; @@ -360,7 +441,8 @@ function buildNextActions(contract: Omit) { function buildContract( projectId: string, normalization: SeoNormalizationContract, - decisionSnapshots: DecisionSnapshot[] = [] + decisionSnapshots: DecisionSnapshot[] = [], + demandCollectionProfile: DemandCollectionProfile = DEFAULT_DEMAND_COLLECTION_PROFILE ): AnchorReviewContract { const decisionSnapshotsById = new Map(decisionSnapshots.map((snapshot) => [snapshot.id, snapshot])); const decisionSnapshotsByKey = new Map(decisionSnapshots.map((snapshot) => [snapshot.key, snapshot])); @@ -369,7 +451,11 @@ function buildContract( ); const seedAnchorIds = new Set(anchors.map((anchor) => anchor.id)); const seedAnchorKeys = new Set(anchors.map((anchor) => anchor.key)); - const allAnchors = [...anchors, ...getSavedManualAnchors(decisionSnapshots, seedAnchorIds, seedAnchorKeys)]; + const allAnchorCandidates = [...anchors, ...getSavedManualAnchors(decisionSnapshots, seedAnchorIds, seedAnchorKeys)]; + const allAnchors = allAnchorCandidates.filter((anchor) => anchor.decision.status !== "deleted"); + const deletedAnchors = allAnchorCandidates + .filter((anchor) => anchor.decision.status === "deleted") + .map(mapDeletedAnchor); const wordstatQueue = buildWordstatQueue(allAnchors); const pendingAnchorCount = allAnchors.filter((anchor) => anchor.decision.status === "pending").length; const approvedAnchorCount = allAnchors.filter((anchor) => anchor.decision.status === "approved").length; @@ -382,6 +468,7 @@ function buildContract( projectOntologyVersionId: normalization.projectOntologyVersionId, generatedAt: new Date().toISOString(), state: getState(normalization, allAnchors, wordstatQueue.length), + demandCollectionProfile, source: { normalizationGeneratedAt: normalization.generatedAt, normalizationProviderMode: normalization.provider.mode, @@ -409,6 +496,55 @@ function buildContract( wordstatQueueCount: wordstatQueue.length }, anchors: allAnchors, + deletedAnchors, + wordstatQueue + }; + + return { + ...contractWithoutActions, + nextActions: buildNextActions(contractWithoutActions) + }; +} + +function rebuildContractWithAnchors(contract: AnchorReviewContract, anchors: AnchorReviewAnchor[]): AnchorReviewContract { + const deletedAnchorById = new Map((contract.deletedAnchors ?? []).map((anchor) => [anchor.id, anchor])); + + anchors + .filter((anchor) => anchor.decision.status === "deleted") + .forEach((anchor) => deletedAnchorById.set(anchor.id, mapDeletedAnchor(anchor))); + + const visibleAnchors = anchors.filter((anchor) => anchor.decision.status !== "deleted"); + const wordstatQueue = buildWordstatQueue(visibleAnchors); + const pendingAnchorCount = visibleAnchors.filter((anchor) => anchor.decision.status === "pending").length; + const approvedAnchorCount = visibleAnchors.filter((anchor) => anchor.decision.status === "approved").length; + const disabledAnchorCount = visibleAnchors.filter((anchor) => anchor.decision.status === "disabled").length; + const manualDecisionCount = visibleAnchors.filter((anchor) => anchor.decision.decidedAt).length; + const contractWithoutActions: Omit = { + ...contract, + generatedAt: new Date().toISOString(), + state: getStateFromContract(contract, visibleAnchors, wordstatQueue.length), + analystReview: { + notes: + wordstatQueue.length > 0 + ? ["Human-approved anchor queue exists. Market collection must use only wordstatQueue."] + : ["Anchor queue awaits human approval before market collection."], + reviewedAt: manualDecisionCount > 0 ? anchors.find((anchor) => anchor.decision.decidedAt)?.decision.decidedAt ?? null : null, + reviewer: manualDecisionCount > 0 ? "human" : "system", + status: wordstatQueue.length > 0 ? "approved" : "not_reviewed" + }, + readiness: { + ...contract.readiness, + approvedAnchorCount, + disabledAnchorCount, + manualDecisionCount, + needsHumanReviewCount: visibleAnchors.filter((anchor) => anchor.proposal.status === "needs_human").length, + normalizedAnchorCount: visibleAnchors.length, + pendingAnchorCount, + proposedWordstatCount: visibleAnchors.filter((anchor) => anchor.proposal.sendToWordstat).length, + wordstatQueueCount: wordstatQueue.length + }, + anchors: visibleAnchors, + deletedAnchors: Array.from(deletedAnchorById.values()), wordstatQueue }; @@ -446,7 +582,12 @@ export async function getAnchorReviewContract(projectId: string): Promise [input.anchorId, input])); + const anchors = contract.anchors.map((anchor) => { + const input = inputsByAnchorId.get(anchor.id); + + if (!input) { + return anchor; + } + + return { + ...anchor, + decision: applyDecisionInput(anchor, input, decidedAt) + }; + }); + + return rebuildContractWithAnchors(contract, anchors); +} + export async function saveAnchorReviewDecisions( projectId: string, - decisions: AnchorReviewDecisionInput[] = [] + decisions?: AnchorReviewDecisionInput[], + demandCollectionProfile?: DemandCollectionProfile ): Promise { const current = await getAnchorReviewContract(projectId); @@ -508,18 +688,25 @@ export async function saveAnchorReviewDecisions( const decidedAt = new Date().toISOString(); const anchorsById = new Map(current.anchors.map((anchor) => [anchor.id, anchor])); const nextDecisionById = new Map(current.anchors.map((anchor) => [anchor.id, anchor.decision])); + const hasExplicitDecisions = Array.isArray(decisions); + const explicitDecisions = decisions ?? []; + const nextDemandCollectionProfile = normalizeDemandCollectionProfile( + demandCollectionProfile ?? current.demandCollectionProfile + ); const inputs = - decisions.length > 0 - ? decisions - : current.anchors - .filter((anchor) => anchor.proposal.sendToWordstat && anchor.proposal.status !== "rejected") - .map((anchor) => ({ - anchorId: anchor.id, - reason: "Human approved proposed Wordstat anchor queue.", - sendToSerp: anchor.proposal.sendToSerp, - sendToWordstat: true, - status: "approved" as const - })); + explicitDecisions.length > 0 + ? explicitDecisions + : !hasExplicitDecisions && demandCollectionProfile === undefined + ? current.anchors + .filter((anchor) => anchor.proposal.sendToWordstat && anchor.proposal.status !== "rejected") + .map((anchor) => ({ + anchorId: anchor.id, + reason: "Human approved proposed Wordstat anchor queue.", + sendToSerp: anchor.proposal.sendToSerp, + sendToWordstat: true, + status: "approved" as const + })) + : []; for (const input of inputs) { const anchor = anchorsById.get(input.anchorId); @@ -531,14 +718,33 @@ export async function saveAnchorReviewDecisions( nextDecisionById.set(anchor.id, applyDecisionInput(anchor, input, decidedAt)); } - const decisionSnapshots: DecisionSnapshot[] = current.anchors.map((anchor) => ({ - anchor, - decision: nextDecisionById.get(anchor.id) ?? anchor.decision, - id: anchor.id, - key: anchor.key - })); + const decisionSnapshots: DecisionSnapshot[] = [ + ...current.anchors.map((anchor) => ({ + anchor, + decision: nextDecisionById.get(anchor.id) ?? anchor.decision, + id: anchor.id, + key: anchor.key + })), + ...(current.deletedAnchors ?? []).map((anchor) => ({ + decision: { + decidedAt: anchor.deletedAt, + decidedBy: "dev-mode", + reason: anchor.reason || "Anchor deleted by review.", + sendToSerp: false, + sendToWordstat: false, + source: "human" as const, + status: "deleted" as const + }, + id: anchor.id, + key: anchor.key + })) + ]; const normalization = await getSeoNormalizationContract(projectId); - const output = buildContract(projectId, normalization, decisionSnapshots); + const output = applyDecisionInputsToContract( + buildContract(projectId, normalization, decisionSnapshots, nextDemandCollectionProfile), + inputs, + decidedAt + ); const result = await pool.query( ` insert into runs (project_id, run_type, status, input, output, started_at, completed_at) @@ -549,6 +755,7 @@ export async function saveAnchorReviewDecisions( projectId, JSON.stringify({ decisionCount: inputs.length, + demandCollectionProfile: output.demandCollectionProfile, schemaVersion: "anchor-review-input.v1", semanticRunId: output.semanticRunId }), @@ -655,6 +862,19 @@ export async function addAnchorReviewManualAnchor( id: anchor.id, key: anchor.key })), + ...(current.deletedAnchors ?? []).map((anchor) => ({ + decision: { + decidedAt: anchor.deletedAt, + decidedBy: "dev-mode", + reason: anchor.reason || "Anchor deleted by review.", + sendToSerp: false, + sendToWordstat: false, + source: "human" as const, + status: "deleted" as const + }, + id: anchor.id, + key: anchor.key + })), { anchor: manualAnchor, decision: manualAnchor.decision, @@ -663,7 +883,7 @@ export async function addAnchorReviewManualAnchor( } ]; const normalization = await getSeoNormalizationContract(projectId); - const output = buildContract(projectId, normalization, decisionSnapshots); + const output = buildContract(projectId, normalization, decisionSnapshots, current.demandCollectionProfile); const result = await pool.query( ` insert into runs (project_id, run_type, status, input, output, started_at, completed_at) diff --git a/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts b/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts new file mode 100644 index 0000000..0f295ad --- /dev/null +++ b/seo_mode/seo_mode/server/src/keywords/keywordAnalysisWorkflow.ts @@ -0,0 +1,768 @@ +import crypto from "node:crypto"; +import { pool } from "../db/client.js"; +import { getAnchorReviewContract } from "./anchorReview.js"; +import { getKeywordCleaningByModelTaskRun, getKeywordCleaningContract } from "./keywordCleaning.js"; +import { getMarketEnrichmentContract, runMarketEnrichment } from "../market/marketEnrichment.js"; +import { getSeoModelProviderStatusContract, runSeoModelTask } from "../modelProvider/seoModelTaskRuns.js"; +import { updateProjectOntologyApproval } from "../ontology/projectOntologyRepository.js"; + +type RunStatus = "queued" | "running" | "done" | "failed" | "cancelled"; +type KeywordAnalysisWorkflowStatus = "blocked" | "completed" | "failed" | "queued" | "running" | "waiting"; +type KeywordAnalysisWorkflowStepId = + | "semantic_basis" + | "project_context" + | "demand_collection" + | "keyword_cleaning" + | "keyword_proposal"; +type KeywordAnalysisWorkflowStepStatus = "blocked" | "completed" | "failed" | "pending" | "running" | "waiting"; + +export type KeywordAnalysisWorkflowStep = { + id: KeywordAnalysisWorkflowStepId; + label: string; + status: KeywordAnalysisWorkflowStepStatus; + detail: string; + reason: string | null; + startedAt: string | null; + completedAt: string | null; +}; + +export type KeywordAnalysisWorkflowContract = { + schemaVersion: "keyword-analysis-workflow.v1"; + runId: string; + projectId: string; + semanticRunId: string | null; + generatedAt: string; + status: KeywordAnalysisWorkflowStatus; + statusLabel: string; + activeStepId: KeywordAnalysisWorkflowStepId | null; + activeStepLabel: string | null; + summary: string; + reason: string | null; + steps: KeywordAnalysisWorkflowStep[]; + artifacts: { + keywordCleaningRunId: string | null; + marketGeneratedAt: string | null; + modelTaskRunId: string | null; + wordstatJobId: string | null; + }; + nextActions: string[]; +}; + +type KeywordAnalysisWorkflowRunRow = { + id: string; + status: RunStatus; + output: KeywordAnalysisWorkflowContract | null; + error_message: string | null; + started_at: Date | null; + completed_at: Date | null; + created_at: Date; +}; + +const WORKFLOW_RUN_TYPE = "keyword_analysis_workflow"; +const activeWorkflowExecutions = new Set(); + +const WORKFLOW_STEPS: Array> = [ + { + detail: "Проверяем, что пользователь выбрал смысловые якоря и разрешил очередь для анализа.", + id: "semantic_basis", + label: "Семантический базис" + }, + { + detail: "Проверяем проектный контекст и готовность онтологии перед внешними данными.", + id: "project_context", + label: "Проектный контекст" + }, + { + detail: "Собираем рыночную базу Wordstat по выбранным якорям, коммерческим формулировкам и смежным веткам.", + id: "demand_collection", + label: "Рыночный спрос" + }, + { + detail: "Передаем собранный рынок в модельную очистку и группировку ключей.", + id: "keyword_cleaning", + label: "Разбор ключей" + }, + { + detail: "Готовим человекочитаемое предложение по ключам для следующего этапа.", + id: "keyword_proposal", + label: "Предложение ключей" + } +]; + +function nowIso() { + return new Date().toISOString(); +} + +function getDemandCollectionProfileLabel(profile: "contextual" | "market_wide") { + return profile === "market_wide" ? "максимальный рыночный охват" : "контекстный охват"; +} + +function isModelTaskRunning(status: string) { + return status === "queued" || status === "running"; +} + +function getWorkflowStatusLabel(status: KeywordAnalysisWorkflowStatus) { + const labels: Record = { + blocked: "Остановлено", + completed: "Анализ готов", + failed: "Ошибка", + queued: "В очереди", + running: "Идет анализ", + waiting: "Ожидаем результат" + }; + + return labels[status]; +} + +function getStepLabel(stepId: KeywordAnalysisWorkflowStepId) { + return WORKFLOW_STEPS.find((step) => step.id === stepId)?.label ?? null; +} + +function buildInitialSteps(): KeywordAnalysisWorkflowStep[] { + return WORKFLOW_STEPS.map((step) => ({ + ...step, + completedAt: null, + reason: null, + startedAt: null, + status: "pending" + })); +} + +function buildInitialWorkflow(projectId: string, runId: string): KeywordAnalysisWorkflowContract { + return { + schemaVersion: "keyword-analysis-workflow.v1", + runId, + projectId, + semanticRunId: null, + generatedAt: nowIso(), + status: "running", + statusLabel: getWorkflowStatusLabel("running"), + activeStepId: "semantic_basis", + activeStepLabel: getStepLabel("semantic_basis"), + summary: "Запускаем анализ выбранных ключей.", + reason: null, + steps: buildInitialSteps(), + artifacts: { + keywordCleaningRunId: null, + marketGeneratedAt: null, + modelTaskRunId: null, + wordstatJobId: null + }, + nextActions: [] + }; +} + +function setWorkflowStatus( + contract: KeywordAnalysisWorkflowContract, + status: KeywordAnalysisWorkflowStatus, + patch: Partial> = {} +): KeywordAnalysisWorkflowContract { + const activeStepId = Object.hasOwn(patch, "activeStepId") ? (patch.activeStepId ?? null) : contract.activeStepId; + + return { + ...contract, + ...patch, + activeStepId, + activeStepLabel: activeStepId ? getStepLabel(activeStepId) : null, + generatedAt: nowIso(), + status, + statusLabel: getWorkflowStatusLabel(status) + }; +} + +function setStepStatus( + contract: KeywordAnalysisWorkflowContract, + stepId: KeywordAnalysisWorkflowStepId, + status: KeywordAnalysisWorkflowStepStatus, + patch: Partial> = {} +): KeywordAnalysisWorkflowContract { + const timestamp = nowIso(); + + return { + ...contract, + activeStepId: status === "completed" ? contract.activeStepId : stepId, + activeStepLabel: status === "completed" ? contract.activeStepLabel : getStepLabel(stepId), + generatedAt: timestamp, + steps: contract.steps.map((step) => { + if (step.id !== stepId) { + return step; + } + + return { + ...step, + ...patch, + completedAt: + status === "blocked" || status === "completed" || status === "failed" ? step.completedAt ?? timestamp : step.completedAt, + reason: Object.hasOwn(patch, "reason") ? (patch.reason ?? null) : step.reason, + startedAt: step.startedAt ?? timestamp, + status + }; + }) + }; +} + +async function saveWorkflow( + contract: KeywordAnalysisWorkflowContract, + runStatus: RunStatus, + errorMessage: string | null = null +) { + const completed = runStatus === "done" || runStatus === "failed" || runStatus === "cancelled"; + const result = await pool.query( + ` + update runs + set status = $2::run_status, + output = $3::jsonb, + error_message = $4, + completed_at = case when $5 then coalesce(completed_at, now()) else completed_at end + where id = $1 and run_type = '${WORKFLOW_RUN_TYPE}' + returning id, status, output, error_message, started_at, completed_at, created_at; + `, + [contract.runId, runStatus, JSON.stringify(contract), errorMessage, completed] + ); + + return mapWorkflowRow(result.rows[0]); +} + +function mapWorkflowRow(row: KeywordAnalysisWorkflowRunRow | undefined) { + return row?.output?.schemaVersion === "keyword-analysis-workflow.v1" ? row.output : null; +} + +async function getWorkflowRun(runId: string) { + const result = await pool.query( + ` + select id, status, output, error_message, started_at, completed_at, created_at + from runs + where id = $1 and run_type = '${WORKFLOW_RUN_TYPE}' + limit 1; + `, + [runId] + ); + + return result.rows[0] ?? null; +} + +async function blockWorkflow( + contract: KeywordAnalysisWorkflowContract, + stepId: KeywordAnalysisWorkflowStepId, + reason: string, + nextActions: string[] = [] +) { + const blocked = setWorkflowStatus(setStepStatus(contract, stepId, "blocked", { reason }), "blocked", { + activeStepId: stepId, + reason, + summary: `${getStepLabel(stepId) ?? "Шаг"} остановлен: ${reason}` + }); + + blocked.nextActions = nextActions.length > 0 ? nextActions : [reason]; + + return saveWorkflow(blocked, "done"); +} + +async function failWorkflow(contract: KeywordAnalysisWorkflowContract, stepId: KeywordAnalysisWorkflowStepId, reason: string) { + const failed = setWorkflowStatus(setStepStatus(contract, stepId, "failed", { reason }), "failed", { + activeStepId: stepId, + reason, + summary: `${getStepLabel(stepId) ?? "Шаг"} завершился ошибкой: ${reason}` + }); + + failed.nextActions = ["Проверить ошибку шага и повторить анализ после исправления причины."]; + + return saveWorkflow(failed, "failed", reason); +} + +async function ensureProjectContextReady(projectId: string) { + let market = await getMarketEnrichmentContract(projectId); + const ontologyVersion = market.projectOntologyVersion; + + if ( + ontologyVersion && + ontologyVersion.approvalStatus === "draft" && + ontologyVersion.reviewProblemCount === 0 + ) { + await updateProjectOntologyApproval({ + approvedBy: "keyword-analysis-workflow", + projectId, + status: "needs_review", + versionId: ontologyVersion.id + }); + market = await getMarketEnrichmentContract(projectId); + } + + return market; +} + +async function ensureMarketFrontierDiscoveryReady( + contract: KeywordAnalysisWorkflowContract, + projectId: string, + demandCollectionProfile: "contextual" | "market_wide" +): Promise<{ contract: KeywordAnalysisWorkflowContract; ready: boolean }> { + if (demandCollectionProfile !== "market_wide") { + return { contract, ready: true }; + } + + if (contract.artifacts.modelTaskRunId) { + const providerStatus = await getSeoModelProviderStatusContract(projectId); + const existingRun = providerStatus.latestRuns.find((run) => run.runId === contract.artifacts.modelTaskRunId); + + if (existingRun && isModelTaskRunning(existingRun.runStatus)) { + const waiting = setWorkflowStatus( + setStepStatus(contract, "demand_collection", "waiting", { + reason: "AI Workspace строит market frontier discovery перед Wordstat." + }), + "waiting", + { + activeStepId: "demand_collection", + reason: "Ждём модельный план рыночных фронтиров.", + summary: "Модель строит память рынка, языковой fingerprint и probe plan перед Wordstat." + } + ); + waiting.nextActions = ["Дождаться завершения seo.market_frontier_discovery; после этого workflow продолжит Wordstat-добор."]; + await saveWorkflow(waiting, "running"); + + return { contract: waiting, ready: false }; + } + + return { contract, ready: true }; + } + + const modelTaskRun = await runSeoModelTask(projectId, { + providerId: "codex_workspace", + taskType: "seo.market_frontier_discovery" + }); + const nextContract = { + ...contract, + artifacts: { + ...contract.artifacts, + modelTaskRunId: modelTaskRun.runId + } + }; + + if (isModelTaskRunning(modelTaskRun.runStatus)) { + const waiting = setWorkflowStatus( + setStepStatus(nextContract, "demand_collection", "waiting", { + reason: "AI Workspace принял market frontier discovery; ждём структурированный probe plan." + }), + "waiting", + { + activeStepId: "demand_collection", + reason: "Ждём модельный план рыночных фронтиров.", + summary: "Модель строит память рынка, языковой fingerprint и probe plan перед Wordstat." + } + ); + waiting.nextActions = ["Дождаться завершения seo.market_frontier_discovery; после этого workflow продолжит Wordstat-добор."]; + await saveWorkflow(waiting, "running"); + + return { contract: waiting, ready: false }; + } + + return { contract: nextContract, ready: true }; +} + +async function completeWorkflow(contract: KeywordAnalysisWorkflowContract) { + const completed = setWorkflowStatus(contract, "completed", { + activeStepId: null, + reason: null, + summary: "Анализ выбранных ключей завершен. Можно разбирать предложение по ключам." + }); + + completed.nextActions = ["Проверить предложенные ключи и перейти к распределению по посадочным."]; + + return saveWorkflow(completed, "done"); +} + +async function executeKeywordAnalysisWorkflow(runId: string, projectId: string) { + if (activeWorkflowExecutions.has(runId)) { + return; + } + + activeWorkflowExecutions.add(runId); + + try { + const row = await getWorkflowRun(runId); + let contract = mapWorkflowRow(row); + + if (!contract) { + return; + } + + contract = setWorkflowStatus(setStepStatus(contract, "semantic_basis", "running"), "running", { + activeStepId: "semantic_basis", + summary: "Проверяем семантический базис." + }); + await saveWorkflow(contract, "running"); + + const anchorReview = await getAnchorReviewContract(projectId); + contract = { + ...contract, + semanticRunId: anchorReview.semanticRunId + }; + + if (!anchorReview.semanticRunId) { + await blockWorkflow(contract, "semantic_basis", "Сначала нужен semantic analysis проекта."); + return; + } + + if (anchorReview.state !== "approved" || anchorReview.readiness.wordstatQueueCount <= 0) { + await blockWorkflow( + contract, + "semantic_basis", + "Нужно выбрать хотя бы один ключ в семантическом базисе и отправить его на анализ." + ); + return; + } + + contract = setStepStatus(contract, "semantic_basis", "completed", { + detail: `Выбрано ${anchorReview.readiness.wordstatQueueCount} ключей для анализа.` + }); + contract = setWorkflowStatus(setStepStatus(contract, "project_context", "running"), "running", { + activeStepId: "project_context", + summary: "Проверяем проектный контекст." + }); + await saveWorkflow(contract, "running"); + + let market = await ensureProjectContextReady(projectId); + contract = { + ...contract, + artifacts: { + ...contract.artifacts, + marketGeneratedAt: market.generatedAt, + wordstatJobId: market.wordstat.latestJob?.id ?? null + } + }; + + if (!market.semanticRunId) { + await blockWorkflow(contract, "project_context", "Сначала нужен semantic analysis проекта.", market.nextActions); + return; + } + + if (!market.readiness.ontologyReadyForMarket) { + await blockWorkflow( + contract, + "project_context", + "Проектный контекст не подтвержден для анализа спроса.", + market.nextActions + ); + return; + } + + contract = setStepStatus(contract, "project_context", "completed", { + detail: "Проектный контекст принят для анализа спроса." + }); + contract = setWorkflowStatus(setStepStatus(contract, "demand_collection", "running"), "running", { + activeStepId: "demand_collection", + summary: "Собираем рыночную базу Wordstat." + }); + await saveWorkflow(contract, "running"); + + if (!market.readiness.canCollectWordstat) { + await blockWorkflow( + contract, + "demand_collection", + "Сбор спроса сейчас недоступен: провайдер или очередь не готовы.", + market.nextActions + ); + return; + } + + const marketFrontierDiscovery = await ensureMarketFrontierDiscoveryReady( + contract, + projectId, + anchorReview.demandCollectionProfile + ); + contract = marketFrontierDiscovery.contract; + + if (!marketFrontierDiscovery.ready) { + return; + } + + const wordstatJobIdBeforeCollection = market.wordstat.latestJob?.id ?? null; + + market = await runMarketEnrichment(projectId); + contract = { + ...contract, + artifacts: { + ...contract.artifacts, + marketGeneratedAt: market.generatedAt, + wordstatJobId: market.wordstat.latestJob?.id ?? null + } + }; + + const latestWordstatJob = market.wordstat.latestJob; + const latestWordstatJobId = latestWordstatJob?.id ?? null; + const hasFreshWordstatFailure = + latestWordstatJob?.status === "failed" && latestWordstatJobId !== wordstatJobIdBeforeCollection; + const hasReusableWordstatEvidence = market.wordstat.summary.resultCount > 0; + + if (hasFreshWordstatFailure || (latestWordstatJob?.status === "failed" && !hasReusableWordstatEvidence)) { + await failWorkflow( + contract, + "demand_collection", + latestWordstatJob.errorMessage ?? "Сбор спроса завершился ошибкой провайдера." + ); + return; + } + + if (!latestWordstatJob) { + await blockWorkflow(contract, "demand_collection", "Сбор спроса не стартовал.", market.nextActions); + return; + } + + contract = setStepStatus(contract, "demand_collection", "completed", { + detail: + `Профиль: ${getDemandCollectionProfileLabel(anchorReview.demandCollectionProfile)}. ` + + `Собрано ${market.wordstat.summary.collectedSeedCount} seed-фраз; ` + + `точный спрос ${market.wordstat.summary.exactFrequencySeedCount}, ` + + `related-сигнал ${market.wordstat.summary.relatedOnlySeedCount}.` + + (market.wordstat.latestJob?.status === "failed" + ? " Последний failed job старый; используем ранее собранное evidence для текущего сравнения." + : "") + }); + contract = setWorkflowStatus(setStepStatus(contract, "keyword_cleaning", "running"), "running", { + activeStepId: "keyword_cleaning", + summary: "Готовим разбор ключей." + }); + await saveWorkflow(contract, "running"); + + const modelTaskRun = await runSeoModelTask(projectId, { taskType: "seo.keyword_cleaning" }); + contract = { + ...contract, + artifacts: { + ...contract.artifacts, + modelTaskRunId: modelTaskRun.runId + } + }; + + if (modelTaskRun.runStatus === "running") { + const waiting = setWorkflowStatus( + setStepStatus(contract, "keyword_cleaning", "waiting", { + reason: "AI Workspace принял задачу; ждем структурированный результат модели." + }), + "waiting", + { + activeStepId: "keyword_cleaning", + reason: "AI Workspace обрабатывает разбор ключей.", + summary: "Ждем модельный разбор ключей." + } + ); + waiting.nextActions = ["Дождаться завершения AI Workspace task; workflow обновится при следующей проверке статуса."]; + await saveWorkflow(waiting, "running"); + return; + } + + if (modelTaskRun.runStatus === "failed") { + const fallbackReason = + modelTaskRun.errorMessage ?? + "Модельный разбор ключей не стартовал, используем backend fallback по собранному Wordstat evidence."; + const fallbackContract = setWorkflowStatus( + setStepStatus(contract, "keyword_cleaning", "running", { + detail: `AI Workspace недоступен: ${fallbackReason}. Собираем backend fallback по exact Wordstat evidence.`, + reason: fallbackReason + }), + "running", + { + activeStepId: "keyword_cleaning", + reason: fallbackReason, + summary: "Модельный разбор недоступен, собираем fallback-разбор ключей." + } + ); + await saveWorkflow(fallbackContract, "running"); + const completedFallback = await finishKeywordCleaningWorkflow(fallbackContract, projectId, null); + + if (!completedFallback) { + await failWorkflow(fallbackContract, "keyword_cleaning", fallbackReason); + } + return; + } + + await finishKeywordCleaningWorkflow(contract, projectId, modelTaskRun.runId); + } catch (error) { + const row = await getWorkflowRun(runId); + const contract = mapWorkflowRow(row); + + if (contract) { + await failWorkflow(contract, contract.activeStepId ?? "keyword_cleaning", error instanceof Error ? error.message : "workflow_failed"); + } + } finally { + activeWorkflowExecutions.delete(runId); + } +} + +async function finishKeywordCleaningWorkflow( + contract: KeywordAnalysisWorkflowContract, + projectId: string, + expectedModelTaskRunId: string | null = contract.artifacts.modelTaskRunId +) { + const keywordCleaningMatch = expectedModelTaskRunId + ? await getKeywordCleaningByModelTaskRun(projectId, expectedModelTaskRunId) + : null; + const keywordCleaning = expectedModelTaskRunId + ? keywordCleaningMatch?.keywordCleaning ?? null + : await getKeywordCleaningContract(projectId); + + if (!keywordCleaning) { + await blockWorkflow( + contract, + "keyword_cleaning", + "Разбор ключей для текущего запуска ещё не сохранён: ждём новый model-task result, старые данные не используются.", + [ + "Дождаться завершения AI Workspace task и повторной проверки workflow.", + "Если task завершился без сохранения keyword cleaning, повторить отправку на анализ." + ] + ); + return null; + } + + if (keywordCleaning.state !== "ready") { + await blockWorkflow( + contract, + "keyword_cleaning", + "Разбор ключей пока не готов: модельный результат не сохранен или не прошел backend-проверку.", + keywordCleaning.nextActions + ); + return null; + } + + let nextContract: KeywordAnalysisWorkflowContract = { + ...contract, + artifacts: { + ...contract.artifacts, + keywordCleaningRunId: keywordCleaningMatch?.runId ?? null + } + }; + + nextContract = setStepStatus(nextContract, "keyword_cleaning", "completed", { + detail: `Разобрано ${keywordCleaning.readiness.sourcePhraseCount} ключей.` + }); + nextContract = setStepStatus(nextContract, "keyword_proposal", "running"); + nextContract = setStepStatus(nextContract, "keyword_proposal", "completed", { + detail: `Предложение готово: ${keywordCleaning.readiness.useCount} основных, ${keywordCleaning.readiness.supportCount} поддерживающих.` + }); + + return completeWorkflow(nextContract); +} + +async function refreshWaitingWorkflow(contract: KeywordAnalysisWorkflowContract) { + if (contract.status !== "waiting" || !contract.artifacts.modelTaskRunId) { + return contract; + } + + const providerStatus = await getSeoModelProviderStatusContract(contract.projectId); + const modelTaskRun = providerStatus.latestRuns.find((run) => run.runId === contract.artifacts.modelTaskRunId) ?? null; + + if (!modelTaskRun || isModelTaskRunning(modelTaskRun.runStatus)) { + return contract; + } + + if (contract.activeStepId === "demand_collection") { + const taskDidNotFinish = modelTaskRun.runStatus === "failed" || modelTaskRun.runStatus === "cancelled"; + const detail = taskDidNotFinish + ? `Market frontier discovery не завершился: ${modelTaskRun.errorMessage ?? "нет provider result"}. Продолжаем через deterministic fallback.` + : "Market frontier discovery готов; продолжаем Wordstat-добор по памяти рынка, fingerprint и probe plan."; + const resumed = setWorkflowStatus( + setStepStatus(contract, "demand_collection", "running", { + detail, + reason: taskDidNotFinish ? modelTaskRun.errorMessage ?? "market_frontier_discovery_not_completed" : null + }), + "running", + { + activeStepId: "demand_collection", + reason: taskDidNotFinish ? modelTaskRun.errorMessage ?? "market_frontier_discovery_not_completed" : null, + summary: taskDidNotFinish + ? "Market frontier discovery недоступен, продолжаем Wordstat через fallback-план." + : "Продолжаем Wordstat-добор по market frontier discovery." + } + ); + resumed.nextActions = ["Workflow продолжит Wordstat-добор; повторно нажимать кнопку не нужно."]; + const saved = await saveWorkflow(resumed, "running"); + void executeKeywordAnalysisWorkflow(contract.runId, contract.projectId); + + return saved ?? resumed; + } + + if (contract.activeStepId !== "keyword_cleaning") { + return contract; + } + + if (modelTaskRun.runStatus === "failed" || modelTaskRun.runStatus === "cancelled") { + const failed = await failWorkflow( + contract, + "keyword_cleaning", + modelTaskRun.errorMessage ?? "Модельный разбор ключей не завершился успешно." + ); + return failed ?? contract; + } + + return (await finishKeywordCleaningWorkflow(contract, contract.projectId)) ?? contract; +} + +export async function getLatestKeywordAnalysisWorkflow(projectId: string): Promise { + const result = await pool.query( + ` + select id, status, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 and run_type = '${WORKFLOW_RUN_TYPE}' + order by created_at desc + limit 1; + `, + [projectId] + ); + const row = result.rows[0] ?? null; + const contract = mapWorkflowRow(row ?? undefined); + + if (!row || !contract) { + return null; + } + + if (row.status === "running" && contract.status === "waiting") { + return refreshWaitingWorkflow(contract); + } + + if (row.status === "running" && (contract.status === "queued" || contract.status === "running") && !activeWorkflowExecutions.has(row.id)) { + void executeKeywordAnalysisWorkflow(row.id, projectId); + } + + return contract; +} + +export async function startKeywordAnalysisWorkflow(projectId: string): Promise { + const existing = await pool.query( + ` + select id, status, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 + and run_type = '${WORKFLOW_RUN_TYPE}' + and status in ('queued', 'running') + order by created_at desc + limit 1; + `, + [projectId] + ); + const existingContract = mapWorkflowRow(existing.rows[0]); + + if (existing.rows[0] && existingContract) { + void executeKeywordAnalysisWorkflow(existing.rows[0].id, projectId); + return existingContract; + } + + const runId = crypto.randomUUID(); + const output = buildInitialWorkflow(projectId, runId); + await pool.query( + ` + insert into runs (id, project_id, run_type, status, input, output, started_at) + values ($1, $2, '${WORKFLOW_RUN_TYPE}', 'running', $3::jsonb, $4::jsonb, now()); + `, + [ + runId, + projectId, + JSON.stringify({ + schemaVersion: "keyword-analysis-workflow-input.v1", + trigger: "semantic_basis_submitted" + }), + JSON.stringify(output) + ] + ); + + void executeKeywordAnalysisWorkflow(runId, projectId); + + return output; +} diff --git a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts index 55a6c8c..e71776f 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordCleaning.ts @@ -1,5 +1,6 @@ import { pool } from "../db/client.js"; import { getMarketEnrichmentContract, type MarketEnrichmentContract } from "../market/marketEnrichment.js"; +import { hasEducationIntentMismatch } from "./keywordIntentGuards.js"; type KeywordCleaningProviderMode = "codex_manual" | "codex_workspace" | "deterministic_fallback"; type KeywordCleaningDecision = "article" | "risky" | "support" | "trash" | "use"; @@ -220,6 +221,11 @@ const CONSUMER_AI_NOISE_PATTERNS = [ const AI_KEYWORD_PATTERN = /(^|\s)(ai|ии)(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект|нейросет/i; const B2B_KEYWORD_FACET_PATTERN = /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|решен|платформ|стоимост|внедрен|купить|заказать|тариф|1с|crm|erp|bpm|workflow|digital|twin|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн|предприят|корпоратив/i; +const KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT = 60; +const KEYWORD_CLEANING_MODEL_LANDING_BRIEF_LIMIT = 18; +const KEYWORD_CLEANING_MODEL_NORMALIZATION_GROUP_LIMIT = 24; +const KEYWORD_CLEANING_MODEL_SEED_QUEUE_LIMIT = 150; +const KEYWORD_CLEANING_MODEL_TOP_RESULT_LIMIT = 120; const ARTICLE_PATTERNS = [ /(^|\s)(как|что такое|что\s+это|почему|зачем|когда|пример|виды|этапы|способы|инструкция|гайд|определен[а-яёa-z0-9-]*|вопрос[а-яёa-z0-9-]*|ответ[а-яёa-z0-9-]*)(\s|$)/i, @@ -258,6 +264,12 @@ function wordCount(value: string) { return normalizePhrase(value).split(/\s+/).filter(Boolean).length; } +function truncateTaskText(value: string | null | undefined, limit = 120) { + const normalized = String(value ?? "").trim().replace(/\s+/g, " "); + + return normalized.length > limit ? `${normalized.slice(0, limit - 1)}…` : normalized; +} + const KEYWORD_CLEANING_FACET_STOP_WORDS = new Set([ "a", "an", @@ -575,7 +587,7 @@ function enforceKeywordCleaningSemanticGuard( }); } -function shouldRescueExactTrashItem(item: KeywordCleaningItem) { +function shouldRescueExactTrashItem(market: MarketEnrichmentContract, item: KeywordCleaningItem) { return ( item.decision === "trash" && item.evidenceStatus === "collected" && @@ -587,6 +599,7 @@ function shouldRescueExactTrashItem(item: KeywordCleaningItem) { !hasPattern(item.normalizedPhrase, ARTICLE_PATTERNS) && !hasPattern(item.normalizedPhrase, RELATED_REVIEW_PATTERNS) && !hasPattern(item.normalizedPhrase, EXACT_TRASH_RESCUE_BLOCKLIST) && + !hasEducationIntentMismatch(market, item.normalizedPhrase) && !isConsumerAiNoisePhrase(item.normalizedPhrase) && !isBroadAiKeywordNoise(item.normalizedPhrase) ); @@ -599,6 +612,7 @@ function isBroadHeadDemandItem(item: KeywordCleaningItem) { } function shouldPromoteExactDemandItem( + market: MarketEnrichmentContract, item: KeywordCleaningItem, contextTokens: Set, guard: KeywordCleaningFacetGuard @@ -621,14 +635,15 @@ function shouldPromoteExactDemandItem( !hasPattern(item.normalizedPhrase, ARTICLE_PATTERNS) && !hasPattern(item.normalizedPhrase, RELATED_REVIEW_PATTERNS) && !hasPattern(item.normalizedPhrase, EXACT_TRASH_RESCUE_BLOCKLIST) && + !hasEducationIntentMismatch(market, item.normalizedPhrase) && !isConsumerAiNoisePhrase(item.normalizedPhrase) && !isBroadAiKeywordNoise(item.normalizedPhrase) ); } -function rescueExactTrashItems(items: KeywordCleaningItem[]): KeywordCleaningItem[] { +function rescueExactTrashItems(market: MarketEnrichmentContract, items: KeywordCleaningItem[]): KeywordCleaningItem[] { return items.map((item) => { - if (!shouldRescueExactTrashItem(item)) { + if (!shouldRescueExactTrashItem(market, item)) { return item; } @@ -648,7 +663,7 @@ function promoteExactDemandItems(market: MarketEnrichmentContract, items: Keywor const guard = buildKeywordCleaningFacetGuard(market); return items.map((item) => { - if (!shouldPromoteExactDemandItem(item, contextTokens, guard)) { + if (!shouldPromoteExactDemandItem(market, item, contextTokens, guard)) { return item; } @@ -667,20 +682,26 @@ function enforceKeywordCleaningBackendGuards( market: MarketEnrichmentContract, items: KeywordCleaningItem[] ): KeywordCleaningItem[] { - return promoteExactDemandItems(market, rescueExactTrashItems(enforceKeywordCleaningSemanticGuard(market, items))); + return promoteExactDemandItems(market, rescueExactTrashItems(market, enforceKeywordCleaningSemanticGuard(market, items))); } function getKeywordCleaningMergeKeys(item: KeywordCleaningItem) { return [item.wordstatResultId ? `wordstat:${item.wordstatResultId}` : null, `phrase:${item.normalizedPhrase}`].filter(Boolean); } -function shouldBackfillMissingExactItem(item: KeywordCleaningItem) { +function shouldBackfillMissingExactItem(market: MarketEnrichmentContract, item: KeywordCleaningItem) { return ( - item.decision !== "trash" && + (item.decision === "use" || item.decision === "support") && item.evidenceStatus === "collected" && item.frequency !== null && Boolean(item.wordstatResultId) && - wordCount(item.phrase) > 1 + wordCount(item.phrase) > 1 && + !hasPattern(item.normalizedPhrase, ARTICLE_PATTERNS) && + !hasPattern(item.normalizedPhrase, RELATED_REVIEW_PATTERNS) && + !hasPattern(item.normalizedPhrase, EXACT_TRASH_RESCUE_BLOCKLIST) && + !hasEducationIntentMismatch(market, item.normalizedPhrase) && + !isConsumerAiNoisePhrase(item.normalizedPhrase) && + !isBroadAiKeywordNoise(item.normalizedPhrase) ); } @@ -690,7 +711,7 @@ function mergeMissingExactEvidenceItems( ): KeywordCleaningItem[] { const knownKeys = new Set(modelItems.flatMap(getKeywordCleaningMergeKeys)); const missingItems = classifyItems(market, getBaseItems(market)) - .filter(shouldBackfillMissingExactItem) + .filter((item) => shouldBackfillMissingExactItem(market, item)) .filter((item) => getKeywordCleaningMergeKeys(item).every((key) => !knownKeys.has(key))) .sort((left, right) => { const leftPriority = left.priority === "high" ? 3 : left.priority === "medium" ? 2 : 1; @@ -698,11 +719,11 @@ function mergeMissingExactEvidenceItems( return rightPriority - leftPriority || (right.frequency ?? 0) - (left.frequency ?? 0); }) - .slice(0, 80) + .slice(0, 24) .map((item) => ({ ...item, evidenceRefs: unique([...item.evidenceRefs, "backend:missing-exact-evidence-backfill"]), - reason: `Backend evidence добавил exact Wordstat item, который модель не вернула в keyword_cleaning. ${item.reason}` + reason: `Backend audit сохранил high-fit exact Wordstat evidence, который модель не вернула в keyword_cleaning. ${item.reason}` })); return [...modelItems, ...missingItems]; @@ -886,13 +907,13 @@ function getBaseItems(contract: MarketEnrichmentContract) { const briefPhraseMap = buildBriefPhraseMap(contract); const briefClusterMap = buildBriefClusterMap(contract); const seedItems = contract.seedQueue - .slice(0, 160) + .slice(0, 260) .map((seed, index) => buildSeedItem(seed, index, briefPhraseMap, briefClusterMap)); const seedByPhrase = new Map(contract.seedQueue.map((seed) => [normalizePhrase(seed.phrase), seed])); const seen = new Set(seedItems.map((item) => item.normalizedPhrase)); const relatedItems: KeywordCleaningItem[] = []; - for (const [index, result] of contract.wordstat.topResults.slice(0, 120).entries()) { + for (const [index, result] of contract.wordstat.topResults.slice(0, 220).entries()) { const normalizedResult = normalizePhrase(result.phrase); if (seen.has(normalizedResult)) { @@ -965,41 +986,84 @@ function mergeModelProviderItemsWithMarketEvidence( }); } +function getKeywordCleaningFrequencyScore(frequency: number | null) { + if (frequency === null) return 0; + if (frequency >= 5000) return 42; + if (frequency >= 1000) return 34; + if (frequency >= 100) return 26; + if (frequency >= 20) return 18; + if (frequency > 0) return 10; + return 0; +} + +function getKeywordCleaningModelSeedScore(seed: SeedQueueItem) { + return ( + getKeywordCleaningFrequencyScore(seed.frequency) + + (seed.evidenceStatus === "collected" ? 36 : 0) + + (seed.wordstatResultId ? 24 : 0) + + (seed.normalization?.marketRole === "commercial" ? 14 : 0) + + (seed.normalization?.marketRole === "core" ? 10 : 0) + + (seed.priority === "high" ? 12 : seed.priority === "medium" ? 6 : 0) + + Math.min(seed.relatedCount ?? 0, 10) + ); +} + +function getKeywordCleaningModelTopResultScore(result: WordstatTopResult) { + return ( + getKeywordCleaningFrequencyScore(result.frequency) + + (result.frequency !== null ? 30 : 0) + + (result.sourcePhrase ? 10 : 0) + + (result.sourceClusterId ? 6 : 0) + + (hasB2BKeywordFacet(result.phrase) ? 8 : 0) - + (isConsumerAiNoisePhrase(result.phrase) ? 50 : 0) + ); +} + function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): KeywordCleaningModelTaskContract["input"] { const briefPhraseMap = buildBriefPhraseMap(contract); const briefClusterMap = buildBriefClusterMap(contract); + const modelSeedQueue = contract.seedQueue + .map((seed, index) => ({ index, score: getKeywordCleaningModelSeedScore(seed), seed })) + .sort((left, right) => right.score - left.score || left.index - right.index) + .slice(0, KEYWORD_CLEANING_MODEL_SEED_QUEUE_LIMIT) + .map((item) => item.seed); + const modelTopResults = contract.wordstat.topResults + .map((result, index) => ({ index, result, score: getKeywordCleaningModelTopResultScore(result) })) + .sort((left, right) => right.score - left.score || left.index - right.index) + .slice(0, KEYWORD_CLEANING_MODEL_TOP_RESULT_LIMIT) + .map((item) => item.result); return { - approvedAnchors: contract.anchorReview.wordstatQueue.slice(0, 80).map((anchor) => ({ + approvedAnchors: contract.anchorReview.wordstatQueue.slice(0, KEYWORD_CLEANING_MODEL_APPROVED_ANCHOR_LIMIT).map((anchor) => ({ clusterId: anchor.clusterId, - clusterTitle: anchor.clusterTitle, + clusterTitle: truncateTaskText(anchor.clusterTitle), confidence: anchor.confidence, intentType: anchor.intentType, marketRole: anchor.marketRole, - normalizedFrom: anchor.normalizedFrom, + normalizedFrom: anchor.normalizedFrom.slice(0, 3).map((phrase) => truncateTaskText(phrase)), phrase: anchor.phrase, priority: anchor.priority })), - landingBriefs: contract.briefEvidence.slice(0, 40).map((brief) => ({ - action: brief.action, + landingBriefs: contract.briefEvidence.slice(0, KEYWORD_CLEANING_MODEL_LANDING_BRIEF_LIMIT).map((brief) => ({ + action: truncateTaskText(brief.action, 160), evidenceStatus: brief.evidenceStatus, path: brief.path, priority: brief.priority, qualityScore: brief.qualityScore, - seedPhrases: brief.seedPhrases.slice(0, 10) + seedPhrases: brief.seedPhrases.slice(0, 6).map((phrase) => truncateTaskText(phrase)) })), - normalizationIntentGroups: contract.normalization.intentGroups.slice(0, 40).map((group) => ({ - excludedPhrases: group.excludedPhrases.slice(0, 12).map((item) => item.phrase), + normalizationIntentGroups: contract.normalization.intentGroups.slice(0, KEYWORD_CLEANING_MODEL_NORMALIZATION_GROUP_LIMIT).map((group) => ({ + excludedPhrases: group.excludedPhrases.slice(0, 6).map((item) => truncateTaskText(item.phrase)), id: group.id, intentType: group.intentType, - marketHypothesis: group.marketHypothesis, + marketHypothesis: truncateTaskText(group.marketHypothesis, 180), priority: group.priority, - title: group.title, - wordstatQueries: group.wordstatQueries.slice(0, 12) + title: truncateTaskText(group.title), + wordstatQueries: group.wordstatQueries.slice(0, 6).map((query) => truncateTaskText(query)) })), - seedQueue: contract.seedQueue.slice(0, 120).map((seed) => ({ + seedQueue: modelSeedQueue.map((seed) => ({ clusterId: seed.clusterId, - clusterTitle: seed.clusterTitle, + clusterTitle: truncateTaskText(seed.clusterTitle), evidenceStatus: seed.evidenceStatus, frequency: seed.frequency, frequencyGroup: seed.frequencyGroup, @@ -1013,14 +1077,14 @@ function buildKeywordCleaningModelInput(contract: MarketEnrichmentContract): Key targetPath: getTargetPath(seed, briefPhraseMap, briefClusterMap), wordstatResultId: seed.wordstatResultId })), - wordstatTopResults: contract.wordstat.topResults.slice(0, 120).map((result) => ({ + wordstatTopResults: modelTopResults.map((result) => ({ frequency: result.frequency, frequencyGroup: result.frequencyGroup, id: result.id, phrase: result.phrase, region: result.region, sourceClusterId: result.sourceClusterId, - sourceClusterTitle: result.sourceClusterTitle, + sourceClusterTitle: truncateTaskText(result.sourceClusterTitle), sourcePhrase: result.sourcePhrase })) }; @@ -1046,7 +1110,7 @@ function isBroadAiKeywordNoise(phrase: string | null) { return AI_KEYWORD_PATTERN.test(normalizedPhrase) && !hasB2BKeywordFacet(normalizedPhrase); } -function getClassification(item: KeywordCleaningItem, contextTokens: Set) { +function getClassification(contract: MarketEnrichmentContract, item: KeywordCleaningItem, contextTokens: Set) { const blockers: string[] = []; const exactEvidence = item.frequency !== null; const noExternalSignal = @@ -1110,6 +1174,16 @@ function getClassification(item: KeywordCleaningItem, contextTokens: Set }; } + if (hasEducationIntentMismatch(contract, item.normalizedPhrase)) { + return { + blockers: unique([...blockers, "education_intent_not_in_product_offer"]), + confidence: exactEvidence ? 0.72 : 0.58, + decision: "article" as const, + reason: + "Фраза относится к обучению/курсам, но продуктовая нормализация сайта не содержит образовательного offer; держим как backlog/контент-гипотезу, не как продуктовую карту Stage 3." + }; + } + if (isArticle && exactEvidence) { return { blockers, @@ -1151,7 +1225,7 @@ function classifyItems(contract: MarketEnrichmentContract, items: KeywordCleanin const contextTokens = buildContextTokenSet(contract); return items.map((item) => { - const classification = getClassification(item, contextTokens); + const classification = getClassification(contract, item, contextTokens); return { ...item, @@ -1181,7 +1255,8 @@ function buildReadiness(items: KeywordCleaningItem[], lanes: KeywordCleaningCont (item) => (item.decision === "use" || item.decision === "support") && item.evidenceStatus === "collected" && - item.frequency !== null + item.frequency !== null && + Boolean(item.wordstatResultId) ).length, missingPageBindingCount: items.filter((item) => item.targetPath === null && item.decision !== "trash").length, riskyCount: lanes.risky.length, @@ -1263,9 +1338,11 @@ function buildKeywordCleaningModelTask( "Не придумывать спрос без Wordstat/SERP evidence.", "Stage 5 не выбирает посадочную автоматически: exact-фразы без targetPath можно раскладывать в keyword map, target/landing решает стратегия.", "Не пропускать exact Wordstat items молча: для каждой значимой exact-фразы из seedQueue/topResults вернуть item с use/support/article/risky/trash и причиной.", + "Если модель не вернула exact-фразу, backend может сохранить её только как audit evidence; Stage 3 не должен показывать такие строки пользователю как candidates.", "Stage 5 собирает глобальную карту спроса, а не rewrite текущей посадочной: частотные exact B2B-смежные ветки с сохранённым source-фасетом должны попадать минимум в support/risky, а не исчезать в article только из-за соседнего рынка.", "Article/backlog использовать для информационного контента и гипотез без сильного коммерческого/продуктового спроса; не маркировать article каждую подтверждённую смежную B2B-ветку.", "Отбрасывать бытовой AI/нейросетевой шум: онлайн, бесплатно, тексты, песни, фото, видео, чат, порно и похожие consumer-запросы не являются B2B SEO-кандидатами без source evidence.", + "Образовательный спрос (курсы, обучение, тренинги, сертификаты) не отправлять в product keyword map, если Product Understanding/normalization не подтверждает образовательный offer сайта.", "Не повышать broad/head phrase до use/support, если Wordstat related потерял protected-смысл исходного якоря; глобальный доминантный термин проекта не должен запрещать отдельные подтверждённые ветки спроса.", "Не менять бизнес-вектор сайта; соседние рынки держать как article/backlog proposal.", "Не сохранять rewrite/apply decisions." @@ -1303,7 +1380,7 @@ function buildNextActions(contract: Omit } actions.push( - `В Stage 3 keyword map предсортировать ${contract.readiness.keywordMapCandidateCount} exact-кандидатов в 5 колонок; финальные роли фиксирует пользователь.` + `В Stage 3 keyword map предсортировать ${contract.readiness.keywordMapCandidateCount} кандидатов в 5 колонок; exact-frequency закрепляется строже, финальные роли фиксирует пользователь.` ); return actions; @@ -1467,6 +1544,29 @@ export async function getLatestAlignedKeywordCleaning( return run; } +export async function getKeywordCleaningByModelTaskRun( + projectId: string, + sourceModelTaskRunId: string +): Promise<{ keywordCleaning: KeywordCleaningContract; runId: string } | null> { + const result = await pool.query( + ` + select id, status, input, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 + and run_type = 'keyword_cleaning' + and input->>'sourceModelTaskRunId' = $2 + order by created_at desc + limit 1; + `, + [projectId, sourceModelTaskRunId] + ); + + const row = result.rows[0] ?? null; + const keywordCleaning = row ? mapKeywordCleaningRun(row) : null; + + return row && keywordCleaning ? { keywordCleaning, runId: row.id } : null; +} + export async function getKeywordCleaningContract(projectId: string): Promise { const market = await getMarketEnrichmentContract(projectId); const fallbackContract = buildFallbackContract(projectId, market); diff --git a/seo_mode/seo_mode/server/src/keywords/keywordContextSnapshots.ts b/seo_mode/seo_mode/server/src/keywords/keywordContextSnapshots.ts new file mode 100644 index 0000000..d56e747 --- /dev/null +++ b/seo_mode/seo_mode/server/src/keywords/keywordContextSnapshots.ts @@ -0,0 +1,261 @@ +import { getLatestSemanticAnalysis } from "../analysis/semanticAnalysis.js"; +import { pool } from "../db/client.js"; +import { getMarketEnrichmentContract } from "../market/marketEnrichment.js"; +import { getAnchorReviewContract } from "./anchorReview.js"; +import { getKeywordCleaningContract } from "./keywordCleaning.js"; +import { getKeywordMapContract } from "./keywordMap.js"; + +type KeywordContextSnapshotInput = { + label?: string; + note?: string; + source?: "manual_save" | "expansion_checkpoint"; + curation?: { + roleOverrides?: Array<{ + itemId: string; + role: string; + }>; + suppressedItemIds?: string[]; + }; +}; + +type KeywordContextSnapshotCurationInput = NonNullable; + +export type KeywordContextSnapshotSummary = { + id: string; + label: string; + source: "manual_save" | "expansion_checkpoint"; + createdAt: string; + semanticRunId: string | null; + keywordMapGeneratedAt: string | null; + keywordCount: number; + keywordPhrases: string[]; + suppressedCount: number; + roleOverrideCount: number; +}; + +type KeywordContextSnapshotRow = { + id: string; + input: { + label?: string; + source?: "manual_save" | "expansion_checkpoint"; + }; + output: { + semanticRunId?: string | null; + keywordMap?: { + generatedAt?: string | null; + items?: unknown[]; + }; + curation?: { + roleOverrides?: unknown[]; + suppressedItemIds?: unknown[]; + }; + } | null; + created_at: Date; +}; + +function getSnapshotKeywordPhrases(row: KeywordContextSnapshotRow) { + const items = row.output?.keywordMap?.items; + + if (!Array.isArray(items)) { + return []; + } + + return items + .map((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) { + return null; + } + + const phrase = (item as { phrase?: unknown }).phrase; + + return typeof phrase === "string" && phrase.trim() ? phrase : null; + }) + .filter((phrase): phrase is string => Boolean(phrase)); +} + +function mapSnapshotRow(row: KeywordContextSnapshotRow): KeywordContextSnapshotSummary { + const curation = row.output?.curation; + const keywordPhrases = getSnapshotKeywordPhrases(row); + + return { + id: row.id, + label: row.input.label || `Ключи и спрос · ${row.created_at.toLocaleString("ru-RU")}`, + source: row.input.source ?? "manual_save", + createdAt: row.created_at.toISOString(), + semanticRunId: row.output?.semanticRunId ?? null, + keywordMapGeneratedAt: row.output?.keywordMap?.generatedAt ?? null, + keywordCount: keywordPhrases.length, + keywordPhrases, + roleOverrideCount: Array.isArray(curation?.roleOverrides) ? curation.roleOverrides.length : 0, + suppressedCount: Array.isArray(curation?.suppressedItemIds) ? curation.suppressedItemIds.length : 0 + }; +} + +export async function listKeywordContextSnapshots(projectId: string): Promise { + const result = await pool.query( + ` + select id, input, output, created_at + from runs + where project_id = $1 + and run_type = 'keyword_context_snapshot' + and status = 'done' + order by created_at desc + limit 30; + `, + [projectId] + ); + + return result.rows.map(mapSnapshotRow); +} + +export async function saveKeywordContextSnapshot( + projectId: string, + input: KeywordContextSnapshotInput +): Promise { + const [semanticAnalysis, anchorReview, marketEnrichment, keywordCleaning, keywordMap] = await Promise.all([ + getLatestSemanticAnalysis(projectId), + getAnchorReviewContract(projectId), + getMarketEnrichmentContract(projectId), + getKeywordCleaningContract(projectId), + getKeywordMapContract(projectId) + ]); + const label = input.label?.trim() || `Ключи и спрос · ${new Date().toLocaleString("ru-RU")}`; + const snapshot = { + schemaVersion: "keyword-context-snapshot.v1", + projectId, + createdAt: new Date().toISOString(), + label, + note: input.note?.trim() || null, + source: input.source ?? "manual_save", + semanticRunId: keywordMap.semanticRunId ?? anchorReview.semanticRunId ?? semanticAnalysis?.runId ?? null, + semanticAnalysis, + anchorReview, + marketEnrichment, + keywordCleaning, + keywordMap, + curation: { + roleOverrides: input.curation?.roleOverrides ?? [], + suppressedItemIds: input.curation?.suppressedItemIds ?? [] + }, + replayContract: { + purpose: "Reproduce Stage 5 context for keyword analysis, frontier expansion and model debugging.", + modelMustKnow: [ + "Исходный контекст сайта и semantic analysis.", + "Какие semantic anchors пользователь подтвердил, скрыл или оставил на разборе.", + "Какой профиль сбора спроса выбран.", + "Какие Wordstat evidence уже собраны и какие seed-фразы дали no signal.", + "Какая текущая keyword map раскладка сохранена пользователем." + ], + expansionRule: + "Дополнительное расширение должно выбирать новые frontier seed-фразы из реального Wordstat evidence, не повторяя уже проверенную траекторию и не сбрасывая пользовательскую раскладку." + } + }; + const result = await pool.query( + ` + insert into runs (project_id, run_type, status, input, output, started_at, completed_at) + values ($1, 'keyword_context_snapshot', 'done', $2::jsonb, $3::jsonb, now(), now()) + returning id, input, output, created_at; + `, + [ + projectId, + JSON.stringify({ + label, + schemaVersion: "keyword-context-snapshot-input.v1", + source: input.source ?? "manual_save" + }), + JSON.stringify(snapshot) + ] + ); + const row = result.rows[0]; + + if (!row) { + throw new Error("Не удалось сохранить snapshot ключей и спроса."); + } + + return mapSnapshotRow(row); +} + +export async function renameKeywordContextSnapshot( + projectId: string, + snapshotId: string, + label: string +): Promise { + const normalizedLabel = label.trim(); + + if (!normalizedLabel) { + throw new Error("Нужно указать название snapshot-профиля."); + } + + const result = await pool.query( + ` + update runs + set input = jsonb_set(coalesce(input, '{}'::jsonb), '{label}', to_jsonb($3::text), true), + output = jsonb_set(coalesce(output, '{}'::jsonb), '{label}', to_jsonb($3::text), true), + completed_at = now() + where id = $2 + and project_id = $1 + and run_type = 'keyword_context_snapshot' + and status = 'done' + returning id, input, output, created_at; + `, + [projectId, snapshotId, normalizedLabel] + ); + const row = result.rows[0]; + + if (!row) { + throw new Error("Snapshot-профиль не найден."); + } + + return mapSnapshotRow(row); +} + +export async function updateKeywordContextSnapshotCuration( + projectId: string, + snapshotId: string, + curation: KeywordContextSnapshotCurationInput +): Promise { + const normalizedCuration = { + roleOverrides: curation.roleOverrides ?? [], + savedAt: new Date().toISOString(), + suppressedItemIds: curation.suppressedItemIds ?? [] + }; + const result = await pool.query( + ` + update runs + set output = jsonb_set(coalesce(output, '{}'::jsonb), '{curation}', $3::jsonb, true), + completed_at = now() + where id = $2 + and project_id = $1 + and run_type = 'keyword_context_snapshot' + and status = 'done' + returning id, input, output, created_at; + `, + [projectId, snapshotId, JSON.stringify(normalizedCuration)] + ); + const row = result.rows[0]; + + if (!row) { + throw new Error("Snapshot-профиль не найден."); + } + + return mapSnapshotRow(row); +} + +export async function deleteKeywordContextSnapshot(projectId: string, snapshotId: string) { + const result = await pool.query( + ` + update runs + set status = 'cancelled', + completed_at = now() + where id = $2 + and project_id = $1 + and run_type = 'keyword_context_snapshot' + and status = 'done'; + `, + [projectId, snapshotId] + ); + + if (result.rowCount === 0) { + throw new Error("Snapshot-профиль не найден."); + } +} diff --git a/seo_mode/seo_mode/server/src/keywords/keywordIntentGuards.ts b/seo_mode/seo_mode/server/src/keywords/keywordIntentGuards.ts new file mode 100644 index 0000000..a93a2aa --- /dev/null +++ b/seo_mode/seo_mode/server/src/keywords/keywordIntentGuards.ts @@ -0,0 +1,108 @@ +import type { MarketEnrichmentContract } from "../market/marketEnrichment.js"; + +const EDUCATION_SEARCH_INTENT_PATTERN = + /(^|\s)(курс[а-яёa-z0-9-]*|обучени[еяю]|учебн[а-яёa-z0-9-]*|урок[а-яёa-z0-9-]*|тренинг[а-яёa-z0-9-]*|семинар[а-яёa-z0-9-]*|вебинар[а-яёa-z0-9-]*|сертификат[а-яёa-z0-9-]*|повышени[а-яёa-z0-9-]*\s+квалификац[а-яёa-z0-9-]*|професси[яию]|course|courses|training|academy|bootcamp|certification)(\s|$)/i; + +const EDUCATION_OFFER_CONTEXT_PATTERN = + /(^|\s)(курс[а-яёa-z0-9-]*|образовательн[а-яёa-z0-9-]*|учебн[а-яёa-z0-9-]*\s+(центр|программ|курс)|академи[яи]|школа|тренинг[а-яёa-z0-9-]*|семинар[а-яёa-z0-9-]*|вебинар[а-яёa-z0-9-]*|сертификат[а-яёa-z0-9-]*|повышени[а-яёa-z0-9-]*\s+квалификац[а-яёa-z0-9-]*|наставничеств[а-яёa-z0-9-]*|практикум[а-яёa-z0-9-]*|обучени[а-яёa-z0-9-]*\s+(сотрудник[а-яёa-z0-9-]*|специалист[а-яёa-z0-9-]*|пользовател[а-яёa-z0-9-]*|клиент[а-яёa-z0-9-]*|команд[а-яёa-z0-9-]*|персонал[а-яёa-z0-9-]*|работе|по)|course|courses|training|academy|bootcamp|certification)(\s|$)/i; + +const MODEL_TRAINING_CONTEXT_PATTERN = + /(^|\s)(машинн[а-яёa-z0-9-]*\s+обучени[а-яёa-z0-9-]*|обучени[а-яёa-z0-9-]*\s+(модел[а-яёa-z0-9-]*|нейросет[а-яёa-z0-9-]*|алгоритм[а-яёa-z0-9-]*|model|models|algorithm[а-яёa-z0-9-]*))(\s|$)/gi; + +function normalizeIntentText(value: string) { + return value.trim().replace(/\s+/g, " ").toLowerCase(); +} + +function pushText(parts: string[], value: string | null | undefined) { + const normalized = normalizeIntentText(String(value ?? "")); + + if (normalized) { + parts.push(normalized); + } +} + +function buildProductOfferContextText(contract: MarketEnrichmentContract) { + const parts: string[] = []; + const modelInput = contract.normalization.modelTask?.input; + + pushText(parts, contract.normalization.summary); + + for (const group of contract.normalization.intentGroups) { + pushText(parts, group.title); + pushText(parts, group.marketHypothesis); + + for (const phrase of group.corePhrases) { + pushText(parts, phrase); + } + } + + pushText(parts, modelInput?.contextReview.summary); + pushText(parts, modelInput?.businessSynthesis.summary); + pushText(parts, modelInput?.businessSynthesis.productFrame?.coreOffer); + pushText(parts, modelInput?.businessSynthesis.productFrame?.productCategory); + pushText(parts, modelInput?.businessSynthesis.productFrame?.centralThesis); + pushText(parts, modelInput?.commercialDemand.summary); + pushText(parts, modelInput?.commercialDemand.commercialCore?.coreBuyerProblem); + pushText(parts, modelInput?.commercialDemand.commercialCore?.coreCommercialValue); + pushText(parts, modelInput?.commercialDemand.commercialCore?.coreDifferentiator); + pushText(parts, modelInput?.commercialDemand.commercialCore?.decisionContext); + + for (const pillar of modelInput?.businessSynthesis.corePillars ?? []) { + pushText(parts, pillar.title); + pushText(parts, pillar.role); + pushText(parts, pillar.grounding); + } + + for (const vector of modelInput?.businessSynthesis.applicationVectors ?? []) { + pushText(parts, vector.title); + pushText(parts, vector.relationshipToCore); + pushText(parts, vector.grounding); + pushText(parts, vector.queryDirection); + } + + for (const family of modelInput?.businessSynthesis.demandFamilies ?? []) { + pushText(parts, family.title); + pushText(parts, family.role); + pushText(parts, family.grounding); + pushText(parts, family.rationale); + + for (const phrase of family.phrases) { + pushText(parts, phrase); + } + } + + for (const family of modelInput?.commercialDemand.buyerIntentFamilies ?? []) { + pushText(parts, family.title); + pushText(parts, family.buyerIntent); + pushText(parts, family.commercialAngle); + pushText(parts, family.sourceMeaning); + pushText(parts, family.coreQualifier); + } + + for (const angle of modelInput?.commercialDemand.applicationCommercialAngles ?? []) { + pushText(parts, angle.sourceVector); + pushText(parts, angle.relationshipToCore); + pushText(parts, angle.grounding); + pushText(parts, angle.commercialReframe); + pushText(parts, angle.coreQualifier); + } + + return parts.join(" "); +} + +export function hasEducationSearchIntent(phrase: string) { + const normalizedPhrase = normalizeIntentText(phrase); + const phraseWithoutModelTraining = normalizedPhrase.replace(MODEL_TRAINING_CONTEXT_PATTERN, " "); + + return EDUCATION_SEARCH_INTENT_PATTERN.test(phraseWithoutModelTraining); +} + +export function hasEducationOfferContext(contract: MarketEnrichmentContract) { + const productContext = buildProductOfferContextText(contract).replace(MODEL_TRAINING_CONTEXT_PATTERN, " "); + + return EDUCATION_OFFER_CONTEXT_PATTERN.test(productContext); +} + +export function hasEducationIntentMismatch(contract: MarketEnrichmentContract, phrase: string) { + return hasEducationSearchIntent(phrase) && !hasEducationOfferContext(contract); +} diff --git a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts index 62e42e3..1afbc33 100644 --- a/seo_mode/seo_mode/server/src/keywords/keywordMap.ts +++ b/seo_mode/seo_mode/server/src/keywords/keywordMap.ts @@ -1,6 +1,7 @@ import { pool } from "../db/client.js"; import { getMarketEnrichmentContract, type MarketEnrichmentContract } from "../market/marketEnrichment.js"; import { getKeywordCleaningContract, type KeywordCleaningContract } from "./keywordCleaning.js"; +import { hasEducationIntentMismatch } from "./keywordIntentGuards.js"; type KeywordMapState = "blocked" | "draft"; type KeywordMapRole = "differentiator" | "primary" | "secondary" | "secondary_candidate" | "support" | "validate"; @@ -82,6 +83,15 @@ export type KeywordMapPersistResult = { type KeywordMapItem = KeywordMapContract["items"][number]; type KeywordCleaningItem = KeywordCleaningContract["items"][number]; +type PersistedKeywordMapLayout = { + reason: string | null; + relatedLanding: string | null; + role: KeywordMapRole; + targetPath: string | null; +}; +type KeywordRoleContext = { + market: MarketEnrichmentContract; +}; export type KeywordMapApproveInput = { itemIds?: string[]; @@ -89,12 +99,39 @@ export type KeywordMapApproveInput = { itemId: string; role: KeywordMapRole; }>; + suppressedItemIds?: string[]; }; function isKeywordMapRole(value: string): value is KeywordMapRole { return keywordMapRoles.includes(value as KeywordMapRole); } +function normalizeKeywordMapDecisionPayload(payload: unknown): Record { + if (payload && typeof payload === "object" && !Array.isArray(payload)) { + return payload as Record; + } + + if (typeof payload === "string") { + try { + const parsed = JSON.parse(payload) as unknown; + + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed as Record; + } + } catch { + return {}; + } + } + + return {}; +} + +function getKeywordMapPayloadRole(payload: unknown) { + const role = normalizeKeywordMapDecisionPayload(payload).role; + + return typeof role === "string" && isKeywordMapRole(role) ? role : null; +} + function normalizePhrase(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } @@ -371,6 +408,10 @@ function getKeywordDecisionRouteWeight(decision: KeywordCleaningItem["decision"] return 0; } +function isBackendMissingExactBackfillItem(item: KeywordCleaningItem) { + return item.evidenceRefs.some((ref) => ref === "backend:missing-exact-evidence-backfill"); +} + function canModelRouteKeyword(item: KeywordCleaningItem, guard?: SemanticFacetGuard) { const semanticFit = guard ? getSemanticFitAssessment(item, guard) : null; @@ -384,41 +425,137 @@ function canModelRouteKeyword(item: KeywordCleaningItem, guard?: SemanticFacetGu ); } +function hasProjectFacetForPrimary(item: KeywordCleaningItem, guard: SemanticFacetGuard) { + const semanticFit = getSemanticFitAssessment(item, guard); + + return semanticFit.projectCoverage === null || semanticFit.projectCoverage > 0; +} + +function isKeywordMapReviewCandidate(item: KeywordCleaningItem, guard: SemanticFacetGuard) { + const semanticFit = getSemanticFitAssessment(item, guard); + + return ( + item.decision === "risky" && + item.evidenceStatus === "collected" && + item.frequency !== null && + Boolean(item.projectOntologyVersionId) && + Boolean(item.wordstatResultId) && + semanticFit.label !== "lost_source_facet" + ); +} + +function hasExactWordstatDemandEvidence(item: KeywordCleaningItem) { + return ( + item.evidenceStatus === "collected" && + item.frequency !== null && + Boolean(item.wordstatResultId) + ); +} + +function isKeywordMapDisplayCandidate(item: KeywordCleaningItem, guard: SemanticFacetGuard, roleContext: KeywordRoleContext) { + const semanticFit = getSemanticFitAssessment(item, guard); + + if ( + item.decision === "trash" || + isBackendMissingExactBackfillItem(item) || + !hasExactWordstatDemandEvidence(item) || + !item.projectOntologyVersionId || + semanticFit.label === "lost_source_facet" || + hasEducationIntentMismatch(roleContext.market, item.phrase) + ) { + return false; + } + + if (canModelRouteKeyword(item, guard) || isKeywordMapReviewCandidate(item, guard)) { + return true; + } + + return item.decision === "use" || item.decision === "support"; +} + function isLongTailKeywordCandidate(item: KeywordCleaningItem) { return item.priority === "low" || getKeywordWordCount(item.phrase) >= 5 || (item.frequency !== null && item.frequency < 50); } +function isLowFrequencyOnlyKeywordCandidate(item: KeywordCleaningItem) { + return item.priority === "low" || (item.frequency !== null && item.frequency < 50); +} + +function isCommercialActionKeywordCandidate(item: KeywordCleaningItem) { + const text = normalizePhrase(item.phrase); + + return ( + item.marketRole === "commercial" || + item.intentType === "commercial" || + /(услуг[а-яёa-z0-9-]*|заказать|купить|стоимост[а-яёa-z0-9-]*|цен[а-яёa-z0-9-]*|под\s+ключ|компани[а-яёa-z0-9-]*\s+по|интегратор[а-яёa-z0-9-]*|внедрен[а-яёa-z0-9-]*|разработк[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)/i.test( + text + ) + ); +} + +function isEditorialOrLocalTailKeywordCandidate(item: KeywordCleaningItem) { + const text = normalizePhrase(item.phrase); + + return /(кейс[а-яёa-z0-9-]*|пример[а-яёa-z0-9-]*|риск[а-яёa-z0-9-]*|специалист[а-яёa-z0-9-]*|компани[а-яёa-z0-9-]*|рынок\s+заказн[а-яёa-z0-9-]*|обзор[а-яёa-z0-9-]*|сравнен[а-яёa-z0-9-]*|рейтинг[а-яёa-z0-9-]*|топ\s+|екатеринбург|ростов|москва|санкт|спб|unity|игр[а-яёa-z0-9-]*)/i.test( + text + ); +} + function isClarifyingKeywordCandidate(item: KeywordCleaningItem) { - const text = getKeywordProfileText(item); + const text = normalizePhrase(item.phrase); return ( item.marketRole === "integration" || item.intentType === "deployment_security" || item.intentType === "integration" || /(^|\s)(api|bim|crm|erp|mcp|iot|1с|3d|digital twin|on[-\s]?premise)(\s|$)/i.test(text) || - /(интеграц|архитект|безопас|облак|workspace|документооборот|утп|дифференц|цифров[а-яёa-z0-9-]*\s+двойн|инженер|тендер|закуп|юрид|договор|беспилот|телеметр|строител|эксплуатац)/i.test( + /(интеграц|архитект|безопас|облак|workspace|документооборот|утп|дифференц|цифров[а-яёa-z0-9-]*\s+двойн|инженер|тендер|закуп|юрид|договор|беспилот|телеметр|строител|эксплуатац|промышлен|производств|учет|отчет|логистик|ритейл|медицин)/i.test( text ) ); } -function getKeywordRole(item: KeywordCleaningItem, indexInTarget: number, guard: SemanticFacetGuard): KeywordMapRole { +function getKeywordRole( + item: KeywordCleaningItem, + indexInTarget: number, + guard: SemanticFacetGuard, + roleContext: KeywordRoleContext +): KeywordMapRole { if (!canModelRouteKeyword(item, guard)) { return "validate"; } - if (item.decision === "use" && indexInTarget === 0 && item.priority === "high" && !isLongTailKeywordCandidate(item)) { - return "primary"; + if (hasEducationIntentMismatch(roleContext.market, item.phrase)) { + return "validate"; } - if (isLongTailKeywordCandidate(item)) { - return "support"; + if ( + indexInTarget === 0 && + (item.decision === "use" || item.decision === "support") && + (item.priority === "high" || item.priority === "medium") && + !isLongTailKeywordCandidate(item) && + !isClarifyingKeywordCandidate(item) && + hasProjectFacetForPrimary(item, guard) + ) { + return "primary"; } if (isClarifyingKeywordCandidate(item)) { return "differentiator"; } + if (isEditorialOrLocalTailKeywordCandidate(item)) { + return "support"; + } + + if (isCommercialActionKeywordCandidate(item)) { + return "secondary"; + } + + if (isLowFrequencyOnlyKeywordCandidate(item)) { + return "support"; + } + if (indexInTarget <= 7 && (item.priority === "high" || item.priority === "medium") && item.decision !== "article") { return "secondary"; } @@ -463,15 +600,10 @@ function buildBlockers(contract: MarketEnrichmentContract, cleaning: KeywordClea function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleaningContract, reviewedOntology: boolean) { const targetCounts = new Map(); const semanticFacetGuard = buildSemanticFacetGuard(contract); + const roleContext: KeywordRoleContext = { market: contract }; const seenPhrases = new Set(); const sourceItems = cleaning.items - .filter( - (item) => - item.decision !== "trash" && - item.evidenceStatus === "collected" && - item.frequency !== null && - Boolean(item.wordstatResultId) - ) + .filter((item) => isKeywordMapDisplayCandidate(item, semanticFacetGuard, roleContext)) .sort((left, right) => { const leftRouteWeight = canModelRouteKeyword(left, semanticFacetGuard) ? 1 : 0; const rightRouteWeight = canModelRouteKeyword(right, semanticFacetGuard) ? 1 : 0; @@ -526,7 +658,7 @@ function buildItems(contract: MarketEnrichmentContract, cleaning: KeywordCleanin const rawTargetPath = item.targetPath; const targetKey = rawTargetPath ?? item.clusterId; const indexInTarget = targetCounts.get(targetKey) ?? 0; - const role = getKeywordRole(item, indexInTarget, semanticFacetGuard); + const role = getKeywordRole(item, indexInTarget, semanticFacetGuard, roleContext); const targetPath = role === "secondary_candidate" ? null : rawTargetPath; const relatedLanding = targetPath === null && rawTargetPath ? rawTargetPath : null; @@ -831,6 +963,109 @@ async function getKeywordMapPersistenceSummary( }; } +async function getSuppressedKeywordMapItems( + projectId: string, + semanticRunId: string | null, + projectOntologyVersionId: string | null +) { + const emptySuppressed = { + normalizedPhrases: new Set(), + sourceItemIds: new Set() + }; + + if (!semanticRunId || !projectOntologyVersionId) { + return emptySuppressed; + } + + const result = await pool.query<{ phrase: string | null; source_item_id: string | null }>( + ` + select source_item_id, phrase + from keyword_decisions + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and status = 'suppressed' + and (source_item_id is not null or nullif(trim(phrase), '') is not null); + `, + [projectId, semanticRunId, projectOntologyVersionId] + ); + + return { + normalizedPhrases: new Set( + result.rows + .map((row) => (row.phrase ? normalizePhrase(row.phrase) : null)) + .filter((phrase): phrase is string => Boolean(phrase)) + ), + sourceItemIds: new Set(result.rows.map((row) => row.source_item_id).filter((id): id is string => Boolean(id))) + }; +} + +async function getPersistedKeywordMapLayouts( + projectId: string, + semanticRunId: string | null, + projectOntologyVersionId: string | null +) { + if (!semanticRunId || !projectOntologyVersionId) { + return new Map(); + } + + const result = await pool.query<{ + source_item_id: string; + target_path: string | null; + related_landing: string | null; + reason: string | null; + payload: unknown; + }>( + ` + select + source_item_id, + target_path, + payload->>'relatedLanding' as related_landing, + reason, + payload + from keyword_decisions + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and source_item_id is not null + and ( + (status = 'approved' and approved = true) + or status = 'layout' + ); + `, + [projectId, semanticRunId, projectOntologyVersionId] + ); + const layouts = new Map(); + + for (const row of result.rows) { + const role = getKeywordMapPayloadRole(row.payload); + + if (!role) { + continue; + } + + layouts.set(row.source_item_id, { + reason: row.reason, + relatedLanding: row.related_landing, + role, + targetPath: row.target_path + }); + } + + return layouts; +} + +function applyPersistedKeywordMapLayout(item: KeywordMapItem, layout: PersistedKeywordMapLayout): KeywordMapItem { + const nonRewriteRole = layout.role === "validate" || layout.role === "secondary_candidate"; + + return { + ...item, + relatedLanding: layout.relatedLanding ?? (nonRewriteRole ? (item.relatedLanding ?? item.targetPath) : item.relatedLanding), + role: layout.role, + targetPath: nonRewriteRole ? null : (layout.targetPath ?? item.targetPath ?? item.relatedLanding) + }; +} + function buildNextActions(contract: Omit) { if (contract.blockers.length > 0) { return contract.blockers; @@ -840,7 +1075,7 @@ function buildNextActions(contract: Omit) { if (contract.cleaning.keywordMapCandidateCount === 0) { actions.push( - "Keyword cleaning не дал exact-кандидатов для Stage 5: SEO-план нельзя фиксировать, сначала нужен повторный market evidence или новая seed strategy." + "Keyword cleaning не дал кандидатов для Stage 5: SEO-план нельзя фиксировать, сначала нужен повторный market evidence или новая seed strategy." ); if (contract.readiness.marketEvidenceCount === 0) { @@ -880,12 +1115,23 @@ export async function getKeywordMapContract(projectId: string): Promise { + const persistedLayout = persistedLayouts.get(item.id); + + return persistedLayout ? applyPersistedKeywordMapLayout(item, persistedLayout) : item; + }) + .filter( + (item) => + !suppressedItems.sourceItemIds.has(item.id) && !suppressedItems.normalizedPhrases.has(normalizePhrase(item.phrase)) + ); const contractWithoutActions: Omit = { schemaVersion: "keyword-map.v1", projectId, @@ -935,6 +1181,7 @@ export async function approveKeywordMapDecisions( .filter((override) => isKeywordMapRole(override.role)) .map((override) => [override.itemId, override.role] as const) ); + const suppressedItemIds = new Set(input.suppressedItemIds ?? []); const selectedItems = keywordMap.items .filter((item) => !itemIdFilter || itemIdFilter.has(item.id)) .map((item) => { @@ -942,9 +1189,17 @@ export async function approveKeywordMapDecisions( return roleOverride ? applyKeywordMapRoleOverride(item, roleOverride) : item; }); - const persistableItems = selectedItems.filter(isPersistableKeywordMapItem); + const suppressedItems = selectedItems.filter((item) => suppressedItemIds.has(item.id)); + const persistableItems = selectedItems.filter((item) => !suppressedItemIds.has(item.id) && isPersistableKeywordMapItem(item)); + const canApproveMap = keywordMap.blockers.length === 0; + const approvableItems = canApproveMap ? persistableItems : []; + const layoutItems = selectedItems.filter( + (item) => + !suppressedItemIds.has(item.id) && + (!isPersistableKeywordMapItem(item) || (!canApproveMap && roleOverrideByItemId.has(item.id))) + ); - if (keywordMap.blockers.length > 0 || !keywordMap.semanticRunId || !keywordMap.projectOntologyVersion) { + if (!keywordMap.semanticRunId || !keywordMap.projectOntologyVersion) { return { keywordMap, savedDecisionCount: 0, @@ -960,9 +1215,9 @@ export async function approveKeywordMapDecisions( let savedDecisionCount = 0; let savedMapItemCount = 0; - const activeSourceItemIds = persistableItems.map((item) => item.id); + const activeSourceItemIds = approvableItems.map((item) => item.id); - if (!itemIdFilter) { + if (!itemIdFilter && canApproveMap) { if (activeSourceItemIds.length > 0) { await client.query( ` @@ -1016,7 +1271,166 @@ export async function approveKeywordMapDecisions( } } - for (const item of persistableItems) { + for (const item of suppressedItems) { + await client.query( + ` + update keyword_map_items + set approved = false, updated_at = now() + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and source_item_id = $4 + and approved = true; + `, + [projectId, keywordMap.semanticRunId, keywordMap.projectOntologyVersion.id, item.id] + ); + + await client.query( + ` + insert into keyword_decisions ( + project_id, + semantic_run_id, + project_ontology_version_id, + wordstat_result_id, + source_item_id, + cluster_id, + cluster_title, + target_path, + phrase, + status, + reason, + approved, + evidence_status, + frequency, + frequency_group, + payload, + updated_at + ) + values ($1, $2, $3, $4, $5, $6, $7, null, $8, 'suppressed', $9, false, $10, $11, $12, $13::jsonb, now()) + on conflict (project_id, project_ontology_version_id, source_item_id) + where source_item_id is not null + do update set + semantic_run_id = excluded.semantic_run_id, + wordstat_result_id = excluded.wordstat_result_id, + cluster_id = excluded.cluster_id, + cluster_title = excluded.cluster_title, + target_path = null, + phrase = excluded.phrase, + status = excluded.status, + reason = excluded.reason, + approved = false, + evidence_status = excluded.evidence_status, + frequency = excluded.frequency, + frequency_group = excluded.frequency_group, + payload = excluded.payload, + updated_at = now(); + `, + [ + projectId, + keywordMap.semanticRunId, + keywordMap.projectOntologyVersion.id, + item.wordstatResultId, + item.id, + item.clusterId, + item.clusterTitle, + item.phrase, + "Пользователь удалил фразу из Stage 5 distribution board.", + item.evidenceStatus, + item.frequency, + item.frequencyGroup, + JSON.stringify({ + relatedLanding: item.relatedLanding, + role: item.role, + source: item.source, + priority: item.priority, + normalizedPhrase: normalizePhrase(item.phrase), + suppressedAt: new Date().toISOString() + }) + ] + ); + savedDecisionCount += 1; + } + + for (const item of layoutItems) { + await client.query( + ` + update keyword_map_items + set approved = false, updated_at = now() + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and source_item_id = $4 + and approved = true; + `, + [projectId, keywordMap.semanticRunId, keywordMap.projectOntologyVersion.id, item.id] + ); + + await client.query( + ` + insert into keyword_decisions ( + project_id, + semantic_run_id, + project_ontology_version_id, + wordstat_result_id, + source_item_id, + cluster_id, + cluster_title, + target_path, + phrase, + status, + reason, + approved, + evidence_status, + frequency, + frequency_group, + payload, + updated_at + ) + values ($1, $2, $3, $4, $5, $6, $7, null, $8, 'layout', $9, false, $10, $11, $12, $13::jsonb, now()) + on conflict (project_id, project_ontology_version_id, source_item_id) + where source_item_id is not null + do update set + semantic_run_id = excluded.semantic_run_id, + wordstat_result_id = excluded.wordstat_result_id, + cluster_id = excluded.cluster_id, + cluster_title = excluded.cluster_title, + target_path = null, + phrase = excluded.phrase, + status = excluded.status, + reason = excluded.reason, + approved = false, + evidence_status = excluded.evidence_status, + frequency = excluded.frequency, + frequency_group = excluded.frequency_group, + payload = excluded.payload, + updated_at = now(); + `, + [ + projectId, + keywordMap.semanticRunId, + keywordMap.projectOntologyVersion.id, + item.wordstatResultId, + item.id, + item.clusterId, + item.clusterTitle, + item.phrase, + item.reason, + item.evidenceStatus, + item.frequency, + item.frequencyGroup, + JSON.stringify({ + relatedLanding: item.relatedLanding, + role: item.role, + source: item.source, + priority: item.priority, + projectOntologyVersionId: item.projectOntologyVersionId + }) + ] + ); + savedDecisionCount += 1; + } + + for (const item of approvableItems) { const decisionResult = await client.query<{ id: string }>( ` insert into keyword_decisions ( @@ -1177,7 +1591,7 @@ export async function approveKeywordMapDecisions( keywordMap: await getKeywordMapContract(projectId), savedDecisionCount, savedMapItemCount, - skippedItemCount: selectedItems.length - persistableItems.length + skippedItemCount: selectedItems.length - suppressedItems.length - layoutItems.length - approvableItems.length }; } catch (error) { await client.query("rollback"); diff --git a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts index 80c30f3..21205f0 100644 --- a/seo_mode/seo_mode/server/src/market/marketEnrichment.ts +++ b/seo_mode/seo_mode/server/src/market/marketEnrichment.ts @@ -1,5 +1,6 @@ import { env } from "../config/env.js"; import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js"; +import { pool } from "../db/client.js"; import { getLatestProjectOntologyVersion, getProjectOntologyVersion, @@ -8,6 +9,7 @@ import { import { collectWordstatEvidence, getLatestWordstatEvidence, + getRequestedWordstatPhraseSet, type WordstatEvidence, type WordstatSeedCandidate } from "./wordstatRepository.js"; @@ -18,6 +20,14 @@ import { } from "../keywords/anchorReview.js"; import { getSeoNormalizationContract, type SeoNormalizationContract } from "../normalization/seoNormalization.js"; import { createWordstatProvider } from "./wordstatProvider.js"; +import { + getSeoCommercialDemandContract, + type SeoCommercialDemandContract +} from "../commercial/seoCommercialDemand.js"; +import { + getMarketFrontierDiscoveryContract, + mapMarketFrontierProbeSeedsToWordstatCandidates +} from "./marketFrontierDiscovery.js"; type MarketProviderId = "model" | "serp" | "text_quality" | "wordstat"; type MarketProviderStatus = "connected" | "connected_partial" | "disabled" | "not_configured" | "not_implemented"; @@ -62,6 +72,7 @@ export type MarketEnrichmentContract = { state: MarketEnrichmentState; normalization: SeoNormalizationContract; anchorReview: AnchorReviewContract; + demandCollectionProfile: AnchorReviewContract["demandCollectionProfile"]; readiness: { anchorApprovedCount: number; anchorPendingCount: number; @@ -226,6 +237,36 @@ function normalizePhrase(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } +async function getSuppressedMarketPhraseSet( + projectId: string, + semanticRunId: string | null, + projectOntologyVersionId: string | null +) { + if (!semanticRunId || !projectOntologyVersionId) { + return new Set(); + } + + const result = await pool.query<{ normalized_phrase: string | null; phrase: string | null }>( + ` + select phrase, payload->>'normalizedPhrase' as normalized_phrase + from keyword_decisions + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and status = 'suppressed' + and (nullif(trim(phrase), '') is not null or nullif(trim(payload->>'normalizedPhrase'), '') is not null); + `, + [projectId, semanticRunId, projectOntologyVersionId] + ); + + return new Set( + result.rows + .flatMap((row) => [row.normalized_phrase, row.phrase]) + .map((phrase) => (phrase ? normalizePhrase(phrase) : null)) + .filter((phrase): phrase is string => Boolean(phrase)) + ); +} + const EXPANSION_STOP_PARTS = new Set([ "ai", "ии", @@ -233,11 +274,8 @@ const EXPANSION_STOP_PARTS = new Set([ "api", "bpm", "crm", - "engine", "erp", - "hub", "mcp", - "ops", "3d", "demo", "pilot", @@ -254,21 +292,7 @@ const EXPANSION_STOP_PARTS = new Set([ "система" ]); -const ALLOWED_EXPANSION_LATIN_TOKENS = new Set(["1c", "ai", "api", "bim", "bpm", "crm", "erp", "iot", "workflow"]); - -const EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = [ - [/digital\s+twin|цифров\w*\s+двойн/i, ["цифровой двойник", "цифровые двойники", "digital twin"]], - [/\bbim\b|(^|\s)бим(\s|$)/i, ["bim", "бим"]], - [/1c|1с/i, ["1с", "1c"]], - [/закуп/i, ["закупки", "автоматизация закупок", "управление закупками"]], - [/тендер/i, ["тендеры", "тендерный помощник", "автоматизация тендеров"]], - [/юрид/i, ["юридические процессы", "юридический ассистент", "автоматизация договоров"]], - [/беспилот/i, ["беспилотная техника", "управление беспилотниками", "платформа для беспилотников"]], - [/\biot\b|интернет\s+вещ/i, ["iot", "интернет вещей", "iot платформа"]], - [/телеметр/i, ["телеметрия", "система телеметрии", "платформа телеметрии"]], - [/строител/i, ["строительство", "автоматизация строительства", "строительный контроль"]], - [/эксплуатац/i, ["эксплуатация объектов", "управление эксплуатацией", "автоматизация эксплуатации"]] -]; +const EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = []; const EXPANSION_QUERY_BLOCKLIST = [ /(^|\s)(базов[а-яёa-z0-9-]*|трендов[а-яёa-z0-9-]*|коммерческ[а-яёa-z0-9-]*)\s+интент/i, @@ -276,10 +300,110 @@ const EXPANSION_QUERY_BLOCKLIST = [ /(^|\s)(задач|проверок)\.$/i, /(^|\s)(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)\s+(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)$/i, /^(платформ[а-яёa-z0-9-]*|систем[а-яёa-z0-9-]*|автоматизац[а-яёa-z0-9-]*)\s+(искусственн[а-яёa-z0-9-]*\s+интеллект|ии|ai)(\s|$)/i, + /^(автоматизац[а-яёa-z0-9-]*|применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(выбор|оценк[аи]|ценност[ьи])$/i, + /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(данн[а-яё]*|процесс[а-яё]*|задач[а-яё]*)$/i, /[.!?:]/, /\b[a-z]+\s+[a-z]+\s+[a-z]+\b/i ]; +const SAFE_SHORT_EXPANSION_LATIN_TOKENS = new Set(["1c", "ai", "api", "bim", "bpm", "crm", "erp", "iot", "mcp", "ops"]); +const DEMAND_COLLECTION_MAX_BASE_RUNS = 1; +const DEMAND_COLLECTION_MAX_MARKET_WIDE_RUNS = 1; +const DEMAND_COLLECTION_CONTEXTUAL_PROBE_BUDGET = 8; +const DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET = 12; +const DEMAND_COLLECTION_MARKET_WIDE_ANCHOR_PROBE_BUDGET = 8; +const WORDSTAT_PROBE_MAX_PER_PATTERN_FAMILY = 4; +const WORDSTAT_PROBE_MAX_PER_CLUSTER = 8; + +type WordstatProbeSourceKind = + | "approved_anchor" + | "commercial_demand" + | "market_frontier_discovery" + | "market_anchor_expansion" + | "previous_expansion"; + +type WordstatProbeCandidate = WordstatSeedCandidate & { + intentType?: AnchorReviewWordstatQueueItem["intentType"]; + marketRole?: AnchorReviewWordstatQueueItem["marketRole"]; + normalizedFrom?: string[]; + probeSource: WordstatProbeSourceKind; +}; + +type WordstatProbeAuditStatus = + | "already_collected" + | "already_requested" + | "duplicate_candidate" + | "invalid_phrase" + | "not_selected_budget_or_balance" + | "selected" + | "suppressed"; + +type WordstatProbePlanSelectedItem = { + phrase: string; + normalizedPhrase: string; + clusterId: string; + clusterTitle: string; + priority: "high" | "medium" | "low"; + source: WordstatProbeSourceKind; + score: number; + reason: string; +}; + +type WordstatProbePlanAuditItem = WordstatProbePlanSelectedItem & { + status: WordstatProbeAuditStatus; + statusLabel: string; +}; + +type WordstatProbePlanWarning = { + code: + | "frontier_missing" + | "high_reuse_blocking" + | "low_diversity" + | "low_selected_count" + | "weak_generic_probes"; + level: "info" | "warning"; + message: string; + examples: string[]; +}; + +export type WordstatProbePlanPreviewContract = { + schemaVersion: "wordstat-probe-plan-preview.v1"; + projectId: string; + semanticRunId: string | null; + generatedAt: string; + demandCollectionProfile: AnchorReviewContract["demandCollectionProfile"]; + orchestration: { + mode: "backend_orchestrated_contract_pipeline"; + executionMode: "preview_only"; + modelCanCallWordstat: false; + steps: string[]; + }; + provider: { + wordstatStatus: MarketProviderStatus; + wordstatMode: string; + }; + readiness: { + canExecute: boolean; + blockers: string[]; + probeBudget: number; + candidateCount: number; + selectedCount: number; + }; + memory: { + requestedPhraseCount: number; + collectedPhraseCount: number; + suppressedPhraseCount: number; + frontierSourceTaskId: string | null; + frontierProbeSeedCount: number; + }; + selectedProbes: WordstatProbePlanSelectedItem[]; + warnings: WordstatProbePlanWarning[]; + candidateSourceCounts: Record; + rejectedSummary: Record, number>; + rejectedSamples: WordstatProbePlanAuditItem[]; + nextActions: string[]; +}; + function wordCount(value: string) { return normalizePhrase(value).split(/\s+/).filter(Boolean).length; } @@ -302,13 +426,513 @@ function unique(values: string[]) { return result; } +function interleaveWordstatSeedGroups(groups: T[][], limit: number) { + const result: T[] = []; + const seenPhrases = new Set(); + + for (let index = 0; result.length < limit; index += 1) { + let added = false; + + for (const group of groups) { + const candidate = group[index]; + + if (!candidate) { + continue; + } + + const normalizedPhrase = normalizePhrase(candidate.phrase); + + if (!normalizedPhrase || seenPhrases.has(normalizedPhrase)) { + continue; + } + + seenPhrases.add(normalizedPhrase); + result.push(candidate); + added = true; + + if (result.length >= limit) { + break; + } + } + + if (!added) { + break; + } + } + + return result; +} + +function getPriorityProbeWeight(priority: WordstatSeedCandidate["priority"]) { + if (priority === "high") return 32; + if (priority === "medium") return 20; + return 8; +} + +function getMarketRoleProbeWeight(role: AnchorReviewWordstatQueueItem["marketRole"] | undefined) { + if (role === "commercial") return 30; + if (role === "core") return 26; + if (role === "problem") return 18; + if (role === "integration") return 16; + if (role === "support") return 8; + if (role === "exclude") return -80; + return 10; +} + +function getProbeSourceWeight(source: WordstatProbeSourceKind) { + if (source === "approved_anchor") return 34; + if (source === "market_frontier_discovery") return 28; + if (source === "commercial_demand") return 20; + if (source === "market_anchor_expansion") return 14; + return 10; +} + +function hasCommercialProbeSignal(phrase: string) { + return /(^|\s)(внедрени[ея]|разработк[аи]|интеграц[ияеи]|заказать|заказ|купить|покупк[аи]|стоимость|цена|тариф)(\s|$)|(^|\s)(платформ[а-яё]*|систем[а-яё]*|решени[ея]|сервис[а-яё]*|автоматизац[а-яё]*)(\s|$)|для\s+(бизнеса|предприяти[яй]|компани[йи])/i.test( + phrase + ); +} + +function hasMarketQualifier(phrase: string) { + return /(^|\s)(ии|ai|корпоративн[а-яё]*|enterprise|бизнес[а-яё]*|промышленн[а-яё]*|компани[яйи]|предприяти[еяй]|малого\s+бизнеса|отрасл[а-яё]*|закуп|тендер|документооборот|учет|производств|строительств)(\s|$)/i.test( + phrase + ); +} + +function getWeakGenericProbeReason(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + const words = wordCount(normalizedPhrase); + const genericObject = /(агент[а-яё]*|данн[а-яё]*|процесс[а-яё]*|задач[а-яё]*|систем[а-яё]*|платформ[а-яё]*)/i; + + if (words <= 2 && genericObject.test(normalizedPhrase) && !hasMarketQualifier(normalizedPhrase)) { + return "Короткая generic-фраза без рыночного/отраслевого квалификатора."; + } + + if ( + /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея]|интеграц[ияеи])\s+(агент[а-яё]*|данн[а-яё]*|процесс[а-яё]*|задач[а-яё]*|систем[а-яё]*|платформ[а-яё]*)$/i.test( + normalizedPhrase + ) + ) { + return "Action + generic object без конкретного рынка, продукта или buyer intent."; + } + + return null; +} + +function getProbeMarketQualityBoost(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + (hasMarketQualifier(normalizedPhrase) ? 16 : 0) + + (/(^|\s)(купить|заказать|стоимость|цена|тариф|поставщик|подрядчик)(\s|$)/i.test(normalizedPhrase) ? 14 : 0) + + (/(^|\s)(для\s+бизнеса|для\s+компани[йи]|для\s+предприяти[яй]|корпоративн[а-яё]*)(\s|$)/i.test(normalizedPhrase) ? 12 : 0) + + (/(^|\s)(промышленн[а-яё]*|строительств|закуп|тендер|документооборот|учет|производств)(\s|$)/i.test(normalizedPhrase) ? 10 : 0) + ); +} + +function getProbeGenericPenalty(phrase: string) { + return getWeakGenericProbeReason(phrase) ? -34 : 0; +} + +function getProbePatternFamily(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + if (/(^|\s)(купить|заказать|стоимость|цена|тариф|поставщик|подрядчик)(\s|$)/i.test(normalizedPhrase)) { + return "commercial"; + } + + if (/закуп|тендер/i.test(normalizedPhrase)) { + return "procurement"; + } + + if (/документооборот|учет|отчет/i.test(normalizedPhrase)) { + return "backoffice"; + } + + if (/промышлен|производств|строительств/i.test(normalizedPhrase)) { + return "vertical"; + } + + if (/^(внедрени[ея]|разработк[аи]|интеграц[ияеи]|создани[ея])\s+/i.test(normalizedPhrase)) { + return "implementation"; + } + + if (/автоматизац/i.test(normalizedPhrase)) { + return "automation"; + } + + if (/(^|\s)(ии|ai|агент[а-яё]*|ассистент[а-яё]*)(\s|$)/i.test(normalizedPhrase)) { + return "ai-agents"; + } + + if (/платформ|систем|решени[ея]|сервис/i.test(normalizedPhrase)) { + return "platform"; + } + + return "other"; +} + +function hasAwkwardWordstatProbeSyntax(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + /(^|\s)для\s+(бизнеса|предприяти[а-яё]*|компани[а-яё]*)\s+для\s+(бизнеса|предприяти[а-яё]*|компани[а-яё]*)/i.test( + normalizedPhrase + ) || + /(^|\s)для\s+[а-яёa-z0-9-]+(?:\s+[а-яёa-z0-9-]+)?\s+для\s+/i.test(normalizedPhrase) || + /(^|\s)для\s+(корпоративная|промышленная|цифровая|автоматизированная)\s+/i.test(normalizedPhrase) || + /^(покупк[аи]|заказ|заказать|купить)\s*\/\s*(покупк[аи]|заказ|заказать|купить)/i.test(normalizedPhrase) || + /(^|\s)(engineering\s+core|core\s+engineering)(\s|$)/i.test(normalizedPhrase) || + /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея]|интеграц(?:ия|ии|ию|ией|ие)|покупк[аи]|заказ|заказать|купить)\s+(?!1c$|ai$|api$|bim$|bpm$|crm$|erp$|iot$|mcp$|ops$)[a-z0-9-]+$/i.test( + normalizedPhrase + ) || + /^(заказать|купить)\s+(digital\s+twin|ai|ии)\s+(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*)/i.test( + normalizedPhrase + ) || + /^(заказать|купить|разработк[аи]|создани[ея]|внедрени[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+enterprise\s+(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*|решени[ея])/i.test( + normalizedPhrase + ) || + /^(внедрени[ея]|интеграц(?:ия|ии|ию|ией|ие)|разработк[аи]|создани[ея])\s+(сервис[а-яё]*|платформ[а-яё]*|систем[а-яё]*|решени[яе])\s+(ии|ai)\s+(ассистент|агент|платформа|система|сервис)(\s|$)/i.test( + normalizedPhrase + ) || + /(^|\s)(ассистент|агент)\s+для\s+(бизнес[а-яё]*|процесс[а-яё]*|предприяти[а-яё]*|компани[а-яё]*)\s+(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*)(\s|$)/i.test( + normalizedPhrase + ) || + /(^|\s)(ассистент|агент)\s+для\s+[а-яёa-z0-9-]+\s+[а-яёa-z0-9-]+\s+(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*)\s+для\s+/i.test( + normalizedPhrase + ) || + /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+(цифрового|корпоративного|промышленного|автоматизированного)$/i.test( + normalizedPhrase + ) || + /^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+enterprise$/i.test( + normalizedPhrase + ) || + /^(заказать|купить)\s+(корпоративная|промышленная|цифровая|автоматизированная)\s+/i.test(normalizedPhrase) || + /^(заказать|купить)\s+(сервис|платформа|система|решени[ея])\s+(корпоративная|промышленная|цифровая|автоматизированная)\s+/i.test( + normalizedPhrase + ) || + /^(внедрени[ея]|разработк[аи]|создани[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+(сервис[а-яё]*|платформ[а-яё]*|систем[а-яё]*|решени[ея])\s+(корпоративная|промышленная|цифровая|автоматизированная)\s+/i.test( + normalizedPhrase + ) || + /^(внедрени[ея]|разработк[аи]|создани[ея]|интеграц(?:ия|ии|ию|ией|ие))\s+(ии|ai)\s+(агент|ассистент|платформа|система|сервис)(\s|$)/i.test( + normalizedPhrase + ) + ); +} + +function isPlausibleWordstatProbePhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + const words = wordCount(normalizedPhrase); + + return ( + words >= 2 && + words <= 7 && + /[а-яё]/i.test(normalizedPhrase) && + !["n/a", "na", "null", "undefined", "нет данных", "не применимо"].includes(normalizedPhrase) && + !/[{}[\]<>#$%^*=~`/|]/.test(normalizedPhrase) && + !hasAwkwardWordstatProbeSyntax(normalizedPhrase) && + !EXPANSION_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedPhrase)) && + !hasUnsafeExpansionLatinToken(normalizedPhrase) + ); +} + +function getReusableWordstatEvidencePhraseSet(wordstat: WordstatEvidence) { + return new Set([ + ...wordstat.seedResults.map((result) => normalizePhrase(result.phrase)), + ...wordstat.topResults + .filter((result) => result.frequency !== null) + .map((result) => normalizePhrase(result.phrase)) + ]); +} + +function scoreWordstatProbeCandidate(candidate: WordstatProbeCandidate) { + const normalizedPhrase = normalizePhrase(candidate.phrase); + const words = wordCount(normalizedPhrase); + + if (!isPlausibleWordstatProbePhrase(normalizedPhrase)) { + return Number.NEGATIVE_INFINITY; + } + + return ( + getProbeSourceWeight(candidate.probeSource) + + getPriorityProbeWeight(candidate.priority) + + getMarketRoleProbeWeight(candidate.marketRole) + + Math.round((candidate.confidence ?? 0.55) * 24) + + (words >= 2 && words <= 4 ? 16 : 0) + + (words === 5 ? 8 : 0) + + (hasCommercialProbeSignal(normalizedPhrase) ? 12 : 0) + + getProbeMarketQualityBoost(normalizedPhrase) + + getProbeGenericPenalty(normalizedPhrase) + + (candidate.normalizedFrom?.length ? 4 : 0) + ); +} + +function getWordstatProbeAuditStatusLabel(status: WordstatProbeAuditStatus) { + const labels: Record = { + already_collected: "уже есть evidence", + already_requested: "уже запрашивали", + duplicate_candidate: "дубликат кандидата", + invalid_phrase: "не похоже на Wordstat-фразу", + not_selected_budget_or_balance: "не вошло в лимит/баланс", + selected: "пойдет в Wordstat", + suppressed: "подавлено пользователем" + }; + + return labels[status]; +} + +function mapWordstatProbePlanSelectedItem(candidate: WordstatProbeCandidate, score: number): WordstatProbePlanSelectedItem { + return { + clusterId: candidate.clusterId, + clusterTitle: candidate.clusterTitle, + normalizedPhrase: normalizePhrase(candidate.phrase), + phrase: candidate.phrase, + priority: candidate.priority, + reason: candidate.reason, + score, + source: candidate.probeSource + }; +} + +function getWordstatProbeFamilyCounts(seeds: Array<{ phrase: string }>) { + const familyCounts = new Map(); + + for (const seed of seeds) { + const family = getProbePatternFamily(seed.phrase); + familyCounts.set(family, (familyCounts.get(family) ?? 0) + 1); + } + + return familyCounts; +} + +function selectWordstatProbeSeeds( + candidates: WordstatProbeCandidate[], + wordstat: WordstatEvidence, + limit: number, + blockedPhrases: Set = new Set(), + options: { + initialFamilyCounts?: Map; + maxPerFamily?: number; + } = {} +) { + const reusablePhrases = getReusableWordstatEvidencePhraseSet(wordstat); + const seenPhrases = new Set(); + const maxPerFamily = options.maxPerFamily ?? WORDSTAT_PROBE_MAX_PER_PATTERN_FAMILY; + const familyCounts = new Map(options.initialFamilyCounts ?? []); + const rankedCandidates = candidates + .map((candidate, index) => ({ + candidate, + index, + normalizedPhrase: normalizePhrase(candidate.phrase), + score: scoreWordstatProbeCandidate(candidate) + })) + .filter((item) => { + if (!Number.isFinite(item.score) || !item.normalizedPhrase) { + return false; + } + + if (blockedPhrases.has(item.normalizedPhrase) || reusablePhrases.has(item.normalizedPhrase)) { + return false; + } + + if (seenPhrases.has(item.normalizedPhrase)) { + return false; + } + + seenPhrases.add(item.normalizedPhrase); + return true; + }) + .sort( + (left, right) => + right.score - left.score || + getPriorityProbeWeight(right.candidate.priority) - getPriorityProbeWeight(left.candidate.priority) || + left.index - right.index + ); + const selected: WordstatProbeCandidate[] = []; + const clusterCounts = new Map(); + + for (let maxPerCluster = 1; maxPerCluster <= WORDSTAT_PROBE_MAX_PER_CLUSTER; maxPerCluster += 1) { + for (const item of rankedCandidates) { + if (selected.length >= limit) { + break; + } + + if (selected.some((seed) => normalizePhrase(seed.phrase) === item.normalizedPhrase)) { + continue; + } + + const clusterCount = clusterCounts.get(item.candidate.clusterId) ?? 0; + + if (clusterCount >= maxPerCluster) { + continue; + } + + const family = getProbePatternFamily(item.normalizedPhrase); + const familyCount = familyCounts.get(family) ?? 0; + + if (familyCount >= maxPerFamily) { + continue; + } + + selected.push(item.candidate); + clusterCounts.set(item.candidate.clusterId, clusterCount + 1); + familyCounts.set(family, familyCount + 1); + } + + if (selected.length >= limit) { + break; + } + } + + return selected; +} + +function buildWordstatProbeAudit(input: { + blockedRequestedPhrases: Set; + candidates: WordstatProbeCandidate[]; + selectedSeeds: WordstatProbeCandidate[]; + suppressedPhrases: Set; + wordstat: WordstatEvidence; +}) { + const reusablePhrases = getReusableWordstatEvidencePhraseSet(input.wordstat); + const selectedPhrases = new Set(input.selectedSeeds.map((seed) => normalizePhrase(seed.phrase))); + const seenPhrases = new Set(); + const auditItems: WordstatProbePlanAuditItem[] = []; + + for (const candidate of input.candidates) { + const normalizedPhrase = normalizePhrase(candidate.phrase); + const score = scoreWordstatProbeCandidate(candidate); + let status: WordstatProbeAuditStatus = "not_selected_budget_or_balance"; + + if (!Number.isFinite(score) || !normalizedPhrase) { + status = "invalid_phrase"; + } else if (input.suppressedPhrases.has(normalizedPhrase)) { + status = "suppressed"; + } else if (input.blockedRequestedPhrases.has(normalizedPhrase)) { + status = "already_requested"; + } else if (reusablePhrases.has(normalizedPhrase)) { + status = "already_collected"; + } else if (seenPhrases.has(normalizedPhrase)) { + status = "duplicate_candidate"; + } else if (selectedPhrases.has(normalizedPhrase)) { + status = "selected"; + } + + if (normalizedPhrase && !seenPhrases.has(normalizedPhrase)) { + seenPhrases.add(normalizedPhrase); + } + + auditItems.push({ + ...mapWordstatProbePlanSelectedItem(candidate, Number.isFinite(score) ? score : 0), + status, + statusLabel: getWordstatProbeAuditStatusLabel(status) + }); + } + + return auditItems; +} + +function selectContextualWordstatProbeSeeds( + anchorSeeds: WordstatProbeCandidate[], + wordstat: WordstatEvidence, + blockedPhrases: Set = new Set() +) { + return selectWordstatProbeSeeds(anchorSeeds, wordstat, DEMAND_COLLECTION_CONTEXTUAL_PROBE_BUDGET, blockedPhrases); +} + +function selectMarketWideWordstatProbeSeeds( + candidates: WordstatProbeCandidate[], + wordstat: WordstatEvidence, + blockedPhrases: Set = new Set() +) { + const anchorProbeSeeds = selectWordstatProbeSeeds( + candidates.filter((candidate) => candidate.probeSource === "approved_anchor"), + wordstat, + DEMAND_COLLECTION_MARKET_WIDE_ANCHOR_PROBE_BUDGET, + blockedPhrases + ); + const usedAnchorPhrases = new Set(anchorProbeSeeds.map((seed) => normalizePhrase(seed.phrase))); + const blockedAndUsedPhrases = new Set([...blockedPhrases, ...usedAnchorPhrases]); + const remainingBudget = DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET - anchorProbeSeeds.length; + const anchorFamilyCounts = getWordstatProbeFamilyCounts(anchorProbeSeeds); + const derivedProbeSeeds = + remainingBudget > 0 + ? selectWordstatProbeSeeds( + candidates.filter((candidate) => candidate.probeSource !== "approved_anchor"), + wordstat, + remainingBudget, + blockedAndUsedPhrases, + { initialFamilyCounts: anchorFamilyCounts } + ) + : []; + + return interleaveWordstatSeedGroups([anchorProbeSeeds, derivedProbeSeeds], DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET); +} + +function buildAnchorWordstatProbeSeeds(anchorReview: AnchorReviewContract, suppressedPhrases: Set) { + return anchorReview.wordstatQueue + .filter((seed) => !suppressedPhrases.has(normalizePhrase(seed.phrase))) + .map((seed) => ({ + clusterId: seed.clusterId, + clusterTitle: seed.clusterTitle, + confidence: seed.confidence, + intentType: seed.intentType, + marketRole: seed.marketRole, + normalizedFrom: seed.normalizedFrom, + phrase: seed.phrase, + priority: seed.priority, + probeSource: "approved_anchor", + reason: seed.reason, + source: seed.source + })); +} + +function markWordstatProbeSource( + candidates: WordstatSeedCandidate[], + probeSource: Exclude +) { + return candidates.map((candidate) => ({ + ...candidate, + probeSource + })); +} + +function getExpansionTokenStems(value: string) { + return new Set( + normalizePhrase(value) + .replace(/ё/g, "е") + .split(/[^a-zа-я0-9]+/i) + .map((token) => token.trim()) + .filter((token) => token.length >= 3) + .filter((token) => !EXPANSION_STOP_PARTS.has(token)) + .map((token) => + /^[a-z0-9-]+$/i.test(token) + ? token.toLowerCase() + : token.replace(/(иями|ями|ами|ого|его|ому|ему|ыми|ими|ых|их|ией|иям|ям|ам|ях|ах|ов|ев|ей|ой|ый|ий|ая|яя|ое|ее|ые|ие|ую|юю|ом|ем|а|я|ы|и|у|ю|е|о)$/i, "") + ) + .filter((token) => token.length >= 3) + ); +} + +function countTokenOverlap(left: Set, right: Set) { + let count = 0; + + for (const item of left) { + if (right.has(item)) { + count += 1; + } + } + + return count; +} + function splitExpansionParts(value: string) { return normalizePhrase(value) .replace(/[()]/g, " ") .replace(/\s+(?:и|and)\s+/gi, ",") .split(/\s*(?:,|\/|&|\+|;)\s*/i) .map((part) => part.trim()) - .filter((part) => wordCount(part) >= 2 || /^[a-z0-9-]{2,16}$/i.test(part)) + .filter((part) => wordCount(part) >= 2 || /^[a-z0-9-]{2,16}$/i.test(part) || /^[а-яё]{5,24}$/i.test(part)) .filter((part) => !EXPANSION_STOP_PARTS.has(part)); } @@ -318,22 +942,85 @@ function getExpansionSynonyms(value: string) { function hasSpecificExpansionFacet(phrase: string) { const normalizedPhrase = normalizePhrase(phrase); + const aiTerms = new Set(["ai", "ии", "искусственный", "искусственного", "искусственным", "интеллект"]); + const meaningfulTerms = normalizedPhrase + .split(/\s+/) + .filter((term) => term.length > 2) + .filter((term) => !aiTerms.has(term)) + .filter((term) => !EXPANSION_STOP_PARTS.has(term)); - return /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|1с|crm|erp|bpm|workflow|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн/i.test( - normalizedPhrase - ); + return meaningfulTerms.length > 0; } function hasUnsafeExpansionLatinToken(phrase: string) { return phrase .split(/\s+/) .flatMap((token) => token.match(/[a-z0-9-]+/gi) ?? []) - .some((token) => !ALLOWED_EXPANSION_LATIN_TOKENS.has(token.toLowerCase())); + .some((token) => { + const normalizedToken = token.toLowerCase(); + + if (SAFE_SHORT_EXPANSION_LATIN_TOKENS.has(normalizedToken)) { + return false; + } + + return ( + normalizedToken.length <= 1 || + normalizedToken.length > 14 || + /^(html|css|js|json|src|href|webp|png|jpe?g|svg|txt)$/i.test(normalizedToken) || + (normalizedToken.length <= 2 && !/\d/.test(normalizedToken)) + ); + }); +} + +const COMMERCIAL_HEAD_INFLECTIONS = [ + { pattern: /^платформа(\s|$)/i, genitive: "платформы$1", accusative: "платформу$1" }, + { pattern: /^система(\s|$)/i, genitive: "системы$1", accusative: "систему$1" }, + { pattern: /^автоматизация(\s|$)/i, genitive: "автоматизации$1", accusative: "автоматизацию$1" }, + { pattern: /^интеграция(\s|$)/i, genitive: "интеграции$1", accusative: "интеграцию$1" }, + { pattern: /^разработка(\s|$)/i, genitive: "разработки$1", accusative: "разработку$1" }, + { pattern: /^решение(\s|$)/i, genitive: "решения$1", accusative: "решение$1" }, + { pattern: /^приложение(\s|$)/i, genitive: "приложения$1", accusative: "приложение$1" }, + { pattern: /^модуль(\s|$)/i, genitive: "модуля$1", accusative: "модуль$1" }, + { pattern: /^сервис(\s|$)/i, genitive: "сервиса$1", accusative: "сервис$1" }, + { pattern: /^агент(\s|$)/i, genitive: "агента$1", accusative: "агента$1" }, + { pattern: /^ассистент(\s|$)/i, genitive: "ассистента$1", accusative: "ассистента$1" }, + { pattern: /^контур(\s|$)/i, genitive: "контура$1", accusative: "контур$1" } +] as const; + +const COMMERCIAL_ACTION_START_PATTERN = + /^(внедрение|внедрения|разработка|разработки|интеграция|интеграции|заказать|заказ|купить|покупка|стоимость|цена|услуги)(\s|$)/i; +const COMMERCIAL_ACTION_AFTER_WRAPPER_PATTERN = + /\b(для|под|через)\s+(внедрение|внедрения|разработка|разработки|интеграция|интеграции|заказать|заказ|купить|покупка)(\s|$)/i; +const COMMERCIAL_DOUBLE_ACTION_PATTERN = + /^(внедрение|разработка|интеграция|заказать|сервис|решение для|платформа для)\s+(внедрение|внедрения|разработка|разработки|интеграция|интеграции|заказать|заказ|купить|покупка)(\s|$)/i; + +function inflectCommercialHead(phrase: string, form: "genitive" | "accusative") { + const cleanPhrase = normalizePhrase(phrase); + const match = COMMERCIAL_HEAD_INFLECTIONS.find((item) => item.pattern.test(cleanPhrase)); + + return match ? cleanPhrase.replace(match.pattern, match[form]) : cleanPhrase; +} + +function startsWithCommercialHead(phrase: string) { + const cleanPhrase = normalizePhrase(phrase); + + return COMMERCIAL_HEAD_INFLECTIONS.some((item) => item.pattern.test(cleanPhrase)); +} + +function startsWithCommercialAction(phrase: string) { + return COMMERCIAL_ACTION_START_PATTERN.test(normalizePhrase(phrase)); +} + +function startsWithCommercialTemplate(phrase: string) { + return startsWithCommercialHead(phrase) || startsWithCommercialAction(phrase); } function buildExpansionPhraseVariants(phrase: string, contextText: string) { const variants = new Set(); const cleanPhrase = normalizePhrase(phrase); + const genitivePhrase = inflectCommercialHead(cleanPhrase, "genitive"); + const accusativePhrase = inflectCommercialHead(cleanPhrase, "accusative"); + const hasActionHead = startsWithCommercialAction(cleanPhrase); const hasAiContext = /\bai\b|(^|\s)ии(\s|$)|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(contextText); if (!cleanPhrase || EXPANSION_STOP_PARTS.has(cleanPhrase)) { @@ -342,16 +1029,16 @@ function buildExpansionPhraseVariants(phrase: string, contextText: string) { variants.add(cleanPhrase); - if (!/платформ/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + if (!hasActionHead && !/платформ/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { variants.add(`платформа ${cleanPhrase}`); variants.add(`${cleanPhrase} платформа`); } - if (!/систем/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + if (!hasActionHead && !/систем/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { variants.add(`система ${cleanPhrase}`); } - if (!/автоматизац/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { + if (!hasActionHead && !startsWithCommercialHead(cleanPhrase) && !/автоматизац/i.test(cleanPhrase) && wordCount(cleanPhrase) <= 4) { variants.add(`автоматизация ${cleanPhrase}`); } @@ -360,6 +1047,21 @@ function buildExpansionPhraseVariants(phrase: string, contextText: string) { variants.add(`ai ${cleanPhrase}`); } + if (wordCount(cleanPhrase) <= 5 && !hasActionHead) { + variants.add(`внедрение ${genitivePhrase}`); + variants.add(`${cleanPhrase} для бизнеса`); + variants.add(`${cleanPhrase} для предприятия`); + } + + if ( + wordCount(cleanPhrase) <= 4 && + !hasActionHead && + /(платформ|систем|сервис|модул|приложен|агент|ассистент|интеграц|автоматизац|workflow|процесс|данн|контур)/i.test(cleanPhrase) + ) { + variants.add(`разработка ${genitivePhrase}`); + variants.add(`заказать ${accusativePhrase}`); + } + return [...variants].filter(isSafeWordstatExpansionPhrase); } @@ -371,14 +1073,21 @@ function isSafeWordstatExpansionPhrase(phrase: string) { wordCount(normalizedPhrase) <= 7 && /[а-яё]/i.test(normalizedPhrase) && normalizedPhrase !== "искусственный интеллект" && - !/[{}[\]<>#$%^*=~`]/.test(normalizedPhrase) && + !/[{}[\]<>#$%^*=~`/|]/.test(normalizedPhrase) && + !hasAwkwardWordstatProbeSyntax(normalizedPhrase) && !EXPANSION_QUERY_BLOCKLIST.some((pattern) => pattern.test(normalizedPhrase)) && !hasUnsafeExpansionLatinToken(normalizedPhrase) && + !/\bдля\s+(платформа|система|автоматизация|интеграция|разработка)\b/i.test(normalizedPhrase) && + !/^(внедрение|разработка|интеграция|заказать)\s+(платформа|система|автоматизация|интеграция)\b/i.test(normalizedPhrase) && + !COMMERCIAL_ACTION_AFTER_WRAPPER_PATTERN.test(normalizedPhrase) && + !COMMERCIAL_DOUBLE_ACTION_PATTERN.test(normalizedPhrase) && + !/^(автоматизация|сервис)\s+(платформа|система|автоматизация|интеграция|разработка)(\s|$)/i.test(normalizedPhrase) && + !/^(платформ[а-яё]*|систем[а-яё]*|сервис[а-яё]*)\s+для\s+.*\b\1\b/i.test(normalizedPhrase) && (!/\b(ai|ии)\b|искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) || hasSpecificExpansionFacet(normalizedPhrase)) && !/(^|\s)(система|платформа)\s+(система|платформа)(\s|$)/i.test(normalizedPhrase) && !/искусственн[а-яёa-z0-9-]*\s+интеллект.*искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) && - !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]{1,2}$/i.test(token) && !/^(ai|bim)$/i.test(token)) + !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]$/i.test(token)) ); } @@ -401,7 +1110,281 @@ function buildClusterExpansionPhrases(cluster: SemanticAnalysisRun["clusters"][n ...getExpansionSynonyms(contextText) ]); - return unique(baseParts.flatMap((part) => buildExpansionPhraseVariants(part, contextText))).slice(0, 16); + return unique(baseParts.flatMap((part) => buildExpansionPhraseVariants(part, contextText))).slice(0, 28); +} + +function getCommercialExpansionPhrases(phrase: string, contextText: string) { + const cleanPhrase = normalizePhrase(phrase); + + if (!cleanPhrase) { + return []; + } + + return unique([ + cleanPhrase, + ...buildExpansionPhraseVariants(cleanPhrase, contextText), + ...(wordCount(cleanPhrase) <= 5 ? [ + ...(startsWithCommercialAction(cleanPhrase) ? [] : [ + `внедрение ${inflectCommercialHead(cleanPhrase, "genitive")}`, + `разработка ${inflectCommercialHead(cleanPhrase, "genitive")}`, + `интеграция ${inflectCommercialHead(cleanPhrase, "genitive")}` + ]), + `${cleanPhrase} для бизнеса`, + `${cleanPhrase} для предприятия`, + ...(startsWithCommercialTemplate(cleanPhrase) ? [] : [`платформа для ${cleanPhrase}`]), + ...(startsWithCommercialTemplate(cleanPhrase) ? [] : [`решение для ${cleanPhrase}`]), + ...(startsWithCommercialTemplate(cleanPhrase) || /^сервис(\s|$)/i.test(cleanPhrase) ? [] : [`сервис ${cleanPhrase}`]) + ] : []) + ]).filter(isSafeWordstatExpansionPhrase); +} + +function getExpansionSeedSource( + phrase: string, + analysis: SemanticAnalysisRun, + anchorSeeds: AnchorReviewWordstatQueueItem[] +): Pick { + const phraseTokens = getExpansionTokenStems(phrase); + const anchorCandidate = anchorSeeds + .map((seed) => { + const text = [ + seed.phrase, + seed.clusterTitle, + seed.intentType, + seed.marketRole, + ...seed.normalizedFrom + ].join(" "); + + return { + score: countTokenOverlap(phraseTokens, getExpansionTokenStems(text)), + seed + }; + }) + .sort((left, right) => right.score - left.score)[0]; + + if (anchorCandidate && anchorCandidate.score > 0) { + return { + clusterId: anchorCandidate.seed.clusterId, + clusterTitle: anchorCandidate.seed.clusterTitle, + priority: anchorCandidate.seed.priority, + source: anchorCandidate.seed.source + }; + } + + const clusterCandidate = analysis.clusters + .map((cluster) => { + const text = [ + cluster.title, + cluster.targetIntent, + ...cluster.matchedTerms, + ...cluster.queryExamples, + ...cluster.sourceRefs.map((ref) => ref.title) + ].join(" "); + + return { + cluster, + score: countTokenOverlap(phraseTokens, getExpansionTokenStems(text)) + }; + }) + .sort((left, right) => right.score - left.score)[0]; + + if (clusterCandidate && clusterCandidate.score > 0) { + return { + clusterId: clusterCandidate.cluster.id, + clusterTitle: clusterCandidate.cluster.title, + priority: clusterCandidate.cluster.priority, + source: "detected" + }; + } + + const fallbackAnchor = anchorSeeds[0]; + + if (fallbackAnchor) { + return { + clusterId: fallbackAnchor.clusterId, + clusterTitle: fallbackAnchor.clusterTitle, + priority: fallbackAnchor.priority, + source: fallbackAnchor.source + }; + } + + const fallbackCluster = analysis.clusters[0]; + + return { + clusterId: fallbackCluster?.id ?? "market-expansion", + clusterTitle: fallbackCluster?.title ?? "Market expansion", + priority: fallbackCluster?.priority ?? "medium", + source: "detected" + }; +} + +function pushCommercialDemandSeed(input: { + analysis: SemanticAnalysisRun; + anchorSeeds: AnchorReviewWordstatQueueItem[]; + candidates: WordstatSeedCandidate[]; + contextText: string; + existingPhrases: Set; + phrase: string; + priority: "high" | "medium" | "low"; + reason: string; +}) { + for (const phrase of getCommercialExpansionPhrases(input.phrase, input.contextText)) { + const normalizedPhrase = normalizePhrase(phrase); + + if (input.existingPhrases.has(normalizedPhrase)) { + continue; + } + + const source = getExpansionSeedSource(normalizedPhrase, input.analysis, input.anchorSeeds); + input.existingPhrases.add(normalizedPhrase); + input.candidates.push({ + clusterId: source.clusterId, + clusterTitle: source.clusterTitle, + confidence: input.priority === "high" ? 0.78 : input.priority === "medium" ? 0.66 : 0.54, + phrase: normalizedPhrase, + priority: input.priority, + reason: input.reason, + source: source.source + }); + } +} + +function buildCommercialDemandWordstatSeeds( + analysis: SemanticAnalysisRun, + anchorReview: AnchorReviewContract, + commercialDemand: SeoCommercialDemandContract | null, + market: MarketEnrichmentContract +): WordstatSeedCandidate[] { + if (!commercialDemand) { + return []; + } + + const existingPhrases = new Set([ + ...anchorReview.wordstatQueue.map((seed) => normalizePhrase(seed.phrase)), + ...market.seedQueue.map((seed) => normalizePhrase(seed.phrase)), + ...market.wordstat.seedResults.map((result) => normalizePhrase(result.phrase)), + ...market.wordstat.topResults.map((result) => normalizePhrase(result.phrase)) + ]); + const candidates: WordstatSeedCandidate[] = []; + const baseContext = [ + commercialDemand.summary, + commercialDemand.commercialCore.coreBuyerProblem, + commercialDemand.commercialCore.coreCommercialValue, + commercialDemand.commercialCore.coreDifferentiator, + commercialDemand.commercialCore.buyingTrigger, + commercialDemand.commercialCore.decisionContext + ].join(" "); + + for (const family of commercialDemand.buyerIntentFamilies.slice(0, 18)) { + const contextText = [ + baseContext, + family.title, + family.buyerIntent, + family.commercialAngle, + family.sourceMeaning, + family.coreQualifier, + ...family.queryPatterns + ].join(" "); + + for (const phrase of unique([ + ...family.queryPatterns, + family.title, + family.commercialAngle, + `${family.coreQualifier} ${family.title}`, + `${family.title} ${family.coreQualifier}` + ])) { + pushCommercialDemandSeed({ + analysis, + anchorSeeds: anchorReview.wordstatQueue, + candidates, + contextText, + existingPhrases, + phrase, + priority: family.priority, + reason: `Commercial demand seed: ${family.title}. ${family.commercialAngle}` + }); + } + } + + for (const angle of commercialDemand.applicationCommercialAngles.slice(0, 18)) { + const contextText = [ + baseContext, + angle.sourceVector, + angle.relationshipToCore, + angle.commercialReframe, + angle.coreQualifier, + ...angle.queryPatterns + ].join(" "); + const sourcePhrases = angle.queryPatterns.length > 0 + ? angle.queryPatterns + : [angle.commercialReframe, angle.sourceVector, `${angle.coreQualifier} ${angle.sourceVector}`]; + + for (const phrase of sourcePhrases) { + pushCommercialDemandSeed({ + analysis, + anchorSeeds: anchorReview.wordstatQueue, + candidates, + contextText, + existingPhrases, + phrase, + priority: angle.priority, + reason: `Application demand seed: ${angle.sourceVector}. ${angle.commercialReframe}` + }); + } + } + + return candidates.slice(0, 180); +} + +function buildMarketWideAnchorWordstatSeeds( + analysis: SemanticAnalysisRun, + anchorReview: AnchorReviewContract, + market: MarketEnrichmentContract +): WordstatSeedCandidate[] { + const existingPhrases = new Set([ + ...anchorReview.wordstatQueue.map((seed) => normalizePhrase(seed.phrase)), + ...market.seedQueue.map((seed) => normalizePhrase(seed.phrase)), + ...market.wordstat.seedResults.map((result) => normalizePhrase(result.phrase)), + ...market.wordstat.topResults.map((result) => normalizePhrase(result.phrase)) + ]); + const candidates: WordstatSeedCandidate[] = []; + const clusterById = new Map(analysis.clusters.map((cluster) => [cluster.id, cluster])); + + for (const anchor of anchorReview.wordstatQueue.slice(0, 120)) { + const cluster = clusterById.get(anchor.clusterId); + const contextText = [ + anchor.phrase, + anchor.clusterTitle, + anchor.reason, + ...anchor.normalizedFrom, + cluster?.title, + cluster?.targetIntent, + ...(cluster?.matchedTerms ?? []), + ...(cluster?.queryExamples ?? []), + ...(cluster?.evidenceSnippets.slice(0, 6).flatMap((snippet) => [snippet.title, snippet.text]) ?? []) + ] + .filter((part): part is string => Boolean(part)) + .join(" "); + const phrases = unique([ + ...getCommercialExpansionPhrases(anchor.phrase, contextText), + ...(cluster ? buildClusterExpansionPhrases(cluster, anchor.phrase) : []) + ]); + + for (const phrase of phrases) { + pushCommercialDemandSeed({ + analysis, + anchorSeeds: anchorReview.wordstatQueue, + candidates, + contextText, + existingPhrases, + phrase, + priority: anchor.priority, + reason: + `Market-wide expansion seed: утвержденный якорь "${anchor.phrase}" раскрыт через коммерческие и смежные формулировки текущего смыслового кластера.` + }); + } + } + + return candidates.slice(0, 220); } function buildWordstatSeedMap(wordstat: WordstatEvidence) { @@ -468,44 +1451,87 @@ function buildSeedQueue( anchorReview: AnchorReviewContract, providers: MarketProviderDescriptor[], wordstat: WordstatEvidence, - projectOntologyVersion: MarketProjectOntologyVersion | null + projectOntologyVersion: MarketProjectOntologyVersion | null, + suppressedPhrases: Set = new Set() ) { const wordstatResults = buildWordstatSeedMap(wordstat); const ontologyReady = isOntologyReadyForMarket(projectOntologyVersion); + const seen = new Set(); + const anchorItems = anchorReview.wordstatQueue + .filter((seed) => !suppressedPhrases.has(normalizePhrase(seed.phrase))) + .slice(0, 120) + .map((seed) => { + const targetProviders: MarketProviderId[] = ["wordstat"]; + const result = wordstatResults.get(normalizePhrase(seed.phrase)); + const evidenceStatus = ontologyReady ? getSeedEvidenceStatus(providers, wordstat, result) : "not_collected"; + seen.add(`${normalizePhrase(seed.phrase)}:${seed.clusterId}`); - return anchorReview.wordstatQueue.slice(0, 120).map((seed) => { - const targetProviders: MarketProviderId[] = ["wordstat"]; - const result = wordstatResults.get(normalizePhrase(seed.phrase)); - const evidenceStatus = ontologyReady ? getSeedEvidenceStatus(providers, wordstat, result) : "not_collected"; + return { + phrase: seed.phrase, + clusterId: seed.clusterId, + clusterTitle: seed.clusterTitle, + priority: seed.priority, + source: seed.source, + normalization: { + confidence: seed.confidence, + intentGroupId: seed.intentGroupId, + intentType: seed.intentType, + marketRole: seed.marketRole, + needsHumanReview: false, + normalizedFrom: seed.normalizedFrom, + reviewReason: seed.reason, + reviewStatus: "auto_approved" as const, + sendToSerp: seed.sendToSerp + }, + evidenceStatus, + targetProviders, + blocker: getSeedBlocker(evidenceStatus), + frequency: result?.frequency ?? null, + frequencyGroup: result?.frequencyGroup ?? null, + relatedCount: result?.relatedCount ?? 0, + lastCollectedAt: result?.collectedAt ?? null, + projectOntologyVersionId: projectOntologyVersion?.id ?? null, + wordstatResultId: result?.id ?? null + }; + }); + const expansionItems = wordstat.seedResults + .filter((result) => { + if (suppressedPhrases.has(normalizePhrase(result.phrase))) { + return false; + } - return { - phrase: seed.phrase, - clusterId: seed.clusterId, - clusterTitle: seed.clusterTitle, - priority: seed.priority, - source: seed.source, - normalization: { - confidence: seed.confidence, - intentGroupId: seed.intentGroupId, - intentType: seed.intentType, - marketRole: seed.marketRole, - needsHumanReview: false, - normalizedFrom: seed.normalizedFrom, - reviewReason: seed.reason, - reviewStatus: "auto_approved" as const, - sendToSerp: seed.sendToSerp - }, - evidenceStatus, - targetProviders, - blocker: getSeedBlocker(evidenceStatus), - frequency: result?.frequency ?? null, - frequencyGroup: result?.frequencyGroup ?? null, - relatedCount: result?.relatedCount ?? 0, - lastCollectedAt: result?.collectedAt ?? null, - projectOntologyVersionId: projectOntologyVersion?.id ?? null, - wordstatResultId: result?.id ?? null - }; - }); + const key = `${normalizePhrase(result.phrase)}:${result.clusterId ?? ""}`; + + if (seen.has(key)) { + return false; + } + + seen.add(key); + return true; + }) + .map((result) => { + const evidenceStatus = ontologyReady ? getSeedEvidenceStatus(providers, wordstat, result) : "not_collected"; + + return { + phrase: result.phrase, + clusterId: result.clusterId ?? "market-expansion", + clusterTitle: result.clusterTitle ?? "Market expansion", + priority: result.priority ?? "medium", + source: result.seedSource === "ontology" ? "ontology" as const : "detected" as const, + normalization: null, + evidenceStatus, + targetProviders: ["wordstat" as const], + blocker: getSeedBlocker(evidenceStatus), + frequency: result.frequency, + frequencyGroup: result.frequencyGroup, + relatedCount: result.relatedCount, + lastCollectedAt: result.collectedAt, + projectOntologyVersionId: projectOntologyVersion?.id ?? null, + wordstatResultId: result.id + }; + }); + + return [...anchorItems, ...expansionItems].slice(0, 260); } function shouldExpandSeed(seed: MarketEnrichmentContract["seedQueue"][number]) { @@ -561,7 +1587,7 @@ function buildWordstatExpansionSeeds( const clusterCandidates = candidatesByCluster.get(seed.clusterId); - if (!clusterCandidates || clusterCandidates.length >= 10) { + if (!clusterCandidates || clusterCandidates.length >= 18) { continue; } @@ -579,7 +1605,7 @@ function buildWordstatExpansionSeeds( const diversifiedCandidates: WordstatSeedCandidate[] = []; - for (let index = 0; diversifiedCandidates.length < 72; index += 1) { + for (let index = 0; diversifiedCandidates.length < 140; index += 1) { let added = false; for (const clusterId of clusterOrder) { @@ -592,7 +1618,7 @@ function buildWordstatExpansionSeeds( diversifiedCandidates.push(candidate); added = true; - if (diversifiedCandidates.length >= 72) { + if (diversifiedCandidates.length >= 140) { break; } } @@ -712,21 +1738,28 @@ function buildNextActions(contract: Omit 0) { actions.push( - `Wordstat evidence собрано по approved queue: ${marketSignalCount}/${contract.readiness.seedCount} фраз дали спрос. Следующий слой — keywordService cleaning.` + `Wordstat evidence собрано по рыночной queue: ${marketSignalCount}/${contract.readiness.seedCount} фраз дали спрос или related-сигнал. Следующий слой — keywordService cleaning.` ); } else { actions.push( - "Approved Wordstat queue собран, но спрос не подтверждён. До SEO-плана нужен model/human review спорных фраз и расширение seed strategy." + "Рыночная Wordstat queue собрана, но спрос не подтверждён. До SEO-плана нужен model/human review спорных фраз и следующий виток seed strategy." ); } } else if (contract.wordstat.latestJob?.status === "failed") { actions.push("Проверить Wordstat provider: последний job завершился ошибкой, raw/error сохранены в dev-mode contract."); } else if (contract.readiness.seedCount > 0) { - actions.push(`Запустить Wordstat collection для ${contract.readiness.seedCount} seed-фраз.`); + const probeBudget = + contract.demandCollectionProfile === "market_wide" + ? DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET + : DEMAND_COLLECTION_CONTEXTUAL_PROBE_BUDGET; + + actions.push( + `Запустить Wordstat collection: до ${Math.min(probeBudget, contract.readiness.seedCount)} сильных probe-фраз, затем расширять карту через top/related результаты.` + ); } if (serp?.status !== "connected") { @@ -734,7 +1767,7 @@ function buildNextActions(contract: Omit 0) { - actions.push(`После настройки провайдера отправить ${contract.readiness.seedCount} seed-фраз в Wordstat queue.`); + actions.push("После настройки провайдера отправлять не всю queue, а короткий ранжированный probe-пакет Wordstat."); } actions.push("Хранить внешние результаты как evidence при seed/brief/page, а не смешивать их с deterministic-анализом."); @@ -749,7 +1782,12 @@ export async function getMarketEnrichmentContract(projectId: string): Promise provider.status === "connected").length; const ontologyReadyForMarket = isOntologyReadyForMarket(projectOntologyVersion); @@ -762,6 +1800,7 @@ export async function getMarketEnrichmentContract(projectId: string): Promise = { + approved_anchor: 0, + commercial_demand: 0, + market_anchor_expansion: 0, + market_frontier_discovery: 0, + previous_expansion: 0 + }; + + for (const candidate of candidates) { + counts[candidate.probeSource] += 1; + } + + return counts; +} + +function getRejectedSummary(auditItems: WordstatProbePlanAuditItem[]) { + const summary: Record, number> = { + already_collected: 0, + already_requested: 0, + duplicate_candidate: 0, + invalid_phrase: 0, + not_selected_budget_or_balance: 0, + suppressed: 0 + }; + + for (const item of auditItems) { + if (item.status !== "selected") { + summary[item.status] += 1; + } + } + + return summary; +} + +function buildWordstatProbePlanWarnings(input: { + candidateSourceCounts: Record; + demandCollectionProfile: AnchorReviewContract["demandCollectionProfile"]; + frontierProbeSeedCount: number; + probeBudget: number; + rejectedSummary: Record, number>; + selectedProbes: WordstatProbePlanSelectedItem[]; +}) { + const warnings: WordstatProbePlanWarning[] = []; + const selectedCount = input.selectedProbes.length; + + if ( + input.demandCollectionProfile === "market_wide" && + input.frontierProbeSeedCount === 0 && + input.candidateSourceCounts.market_frontier_discovery === 0 + ) { + warnings.push({ + code: "frontier_missing", + examples: [], + level: "warning", + message: "Нет frontier probes от модели: market-wide сбор опирается только на локальные расширения." + }); + } + + if (selectedCount === 0) { + warnings.push({ + code: "low_selected_count", + examples: [], + level: "warning", + message: "После памяти и фильтров не осталось новых Wordstat probes." + }); + } else if (selectedCount < input.probeBudget) { + warnings.push({ + code: "low_selected_count", + examples: [], + level: selectedCount < Math.min(4, input.probeBudget) ? "warning" : "info", + message: `Выбрано ${selectedCount}/${input.probeBudget}: лимит не добивается слабым или повторным мусором.` + }); + } + + if (input.rejectedSummary.already_requested >= Math.max(12, selectedCount * 2)) { + warnings.push({ + code: "high_reuse_blocking", + examples: [], + level: "info", + message: `Память заблокировала ${input.rejectedSummary.already_requested} уже запрошенных фраз.` + }); + } + + if (selectedCount >= 4) { + const familyCounts = getWordstatProbeFamilyCounts(input.selectedProbes); + const dominantFamily = [...familyCounts.entries()].sort((left, right) => right[1] - left[1])[0] ?? null; + const dominantShare = dominantFamily ? dominantFamily[1] / selectedCount : 0; + + if (familyCounts.size <= 2 || dominantShare >= 0.58) { + warnings.push({ + code: "low_diversity", + examples: input.selectedProbes + .filter((probe) => dominantFamily && getProbePatternFamily(probe.phrase) === dominantFamily[0]) + .slice(0, 5) + .map((probe) => probe.phrase), + level: "warning", + message: "Probe-план всё ещё сосредоточен в узкой языковой семье спроса." + }); + } + } + + const weakGenericProbes = input.selectedProbes + .filter((probe) => Boolean(getWeakGenericProbeReason(probe.phrase))) + .slice(0, 5) + .map((probe) => probe.phrase); + + if (weakGenericProbes.length > 0) { + warnings.push({ + code: "weak_generic_probes", + examples: weakGenericProbes, + level: weakGenericProbes.length >= 3 ? "warning" : "info", + message: "В план попали generic probes без явного рынка, продукта или buyer intent." + }); + } + + return warnings; +} + +function buildWordstatProbePreviewNextActions(input: { + blockers: string[]; + selectedCount: number; +}) { + if (input.blockers.length > 0) { + return input.blockers; + } + + if (input.selectedCount === 0) { + return ["Новых probe-фраз нет: память рынка считает текущие кандидаты уже requested/collected/suppressed или невалидными."]; + } + + return [ + "Preview-only: внешний Wordstat не вызван, лимит не потрачен.", + `Реальный запуск отправит ${input.selectedCount} выбранных probe-фраз и затем отдаст top/related/popular в keyword cleaning.` + ]; +} + +async function collectWordstatEvidenceBatches( + projectId: string, + analysis: SemanticAnalysisRun, + seedCandidates: WordstatSeedCandidate[], + maxRuns: number, + maxSeedsTotal: number +) { + let market = await getMarketEnrichmentContract(projectId); + let remainingSeedBudget = Math.max(0, maxSeedsTotal); + + for (let runIndex = 0; runIndex < maxRuns && remainingSeedBudget > 0; runIndex += 1) { + const previousJobId = market.wordstat.latestJob?.id ?? null; + try { + await collectWordstatEvidence(projectId, analysis, seedCandidates, { maxSeeds: remainingSeedBudget }); + } catch (error) { + const message = error instanceof Error ? error.message : ""; + + if (/Wordstat hourly quota exhausted/i.test(message)) { + break; + } + + throw error; + } + + market = await getMarketEnrichmentContract(projectId); + + if ((market.wordstat.latestJob?.id ?? null) === previousJobId) { + break; + } + + remainingSeedBudget -= market.wordstat.latestJob?.requestedPhraseCount ?? 0; + } + + return market; +} + +export async function getWordstatProbePlanPreview(projectId: string): Promise { + const analysis = await getLatestSemanticAnalysis(projectId); + const projectOntologyVersion = mapMarketOntologyVersion(await getBoundOntologyVersion(projectId, analysis)); + const providers = await buildProviderRegistry(); + const anchorReview = await getAnchorReviewContract(projectId); + const commercialDemand = await getSeoCommercialDemandContract(projectId); + const suppressedPhrases = await getSuppressedMarketPhraseSet( + projectId, + analysis?.runId ?? null, + projectOntologyVersion?.id ?? null + ); + const requestedPhrases = analysis ? await getRequestedWordstatPhraseSet(projectId, analysis.runId) : new Set(); + const blockedProbePhrases = new Set([...requestedPhrases, ...suppressedPhrases]); + const anchorProbeSeeds = buildAnchorWordstatProbeSeeds(anchorReview, suppressedPhrases); + const blockers = getWordstatProbePlanBlockers({ + analysis, + anchorProbeSeedCount: anchorProbeSeeds.length, + anchorReview, + projectOntologyVersion, + providers + }); + const marketBeforeCollection = await getMarketEnrichmentContract(projectId); + const probeBudget = getWordstatProbeBudget(anchorReview.demandCollectionProfile); + const wordstatProvider = providers.find((provider) => provider.id === "wordstat"); + let candidates: WordstatProbeCandidate[] = []; + let selectedSeeds: WordstatProbeCandidate[] = []; + let frontierSourceTaskId: string | null = null; + let frontierProbeSeedCount = 0; + + if (analysis && blockers.length === 0) { + const blockedProbePhrases = new Set([...requestedPhrases, ...suppressedPhrases]); + + if (anchorReview.demandCollectionProfile === "market_wide") { + const marketWidePlan = await buildMarketWideWordstatProbeCandidates({ + analysis, + anchorProbeSeeds, + anchorReview, + commercialDemand, + marketBeforeCollection, + projectId + }); + candidates = marketWidePlan.candidates; + frontierSourceTaskId = marketWidePlan.marketFrontierDiscovery.sourceTaskId; + frontierProbeSeedCount = marketWidePlan.marketFrontierDiscovery.probeSeeds.length; + selectedSeeds = selectMarketWideWordstatProbeSeeds( + candidates, + marketBeforeCollection.wordstat, + blockedProbePhrases + ); + } else { + candidates = anchorProbeSeeds; + selectedSeeds = selectContextualWordstatProbeSeeds( + candidates, + marketBeforeCollection.wordstat, + blockedProbePhrases + ); + } + } + + const auditItems = buildWordstatProbeAudit({ + blockedRequestedPhrases: requestedPhrases, + candidates, + selectedSeeds, + suppressedPhrases, + wordstat: marketBeforeCollection.wordstat + }); + const selectedProbes = selectedSeeds.map((seed) => + mapWordstatProbePlanSelectedItem(seed, scoreWordstatProbeCandidate(seed)) + ); + const candidateSourceCounts = getCandidateSourceCounts(candidates); + const rejectedSummary = getRejectedSummary(auditItems); + const warnings = buildWordstatProbePlanWarnings({ + candidateSourceCounts, + demandCollectionProfile: anchorReview.demandCollectionProfile, + frontierProbeSeedCount, + probeBudget, + rejectedSummary, + selectedProbes + }); + + return { + schemaVersion: "wordstat-probe-plan-preview.v1", + projectId, + semanticRunId: analysis?.runId ?? null, + generatedAt: new Date().toISOString(), + demandCollectionProfile: anchorReview.demandCollectionProfile, + orchestration: { + mode: "backend_orchestrated_contract_pipeline", + executionMode: "preview_only", + modelCanCallWordstat: false, + steps: [ + "Backend reads site/semantic/market memory contracts.", + "Model-provider task proposes market frontier probes as structured JSON.", + "Backend dedupes requested/collected/suppressed phrases and selects a bounded probe batch.", + "Preview stops before Wordstat; only the real workflow can execute external collection." + ] + }, + provider: { + wordstatMode: wordstatProvider?.mode ?? "not_configured", + wordstatStatus: wordstatProvider?.status ?? "not_configured" + }, + readiness: { + blockers, + canExecute: blockers.length === 0 && selectedProbes.length > 0, + candidateCount: candidates.length, + probeBudget, + selectedCount: selectedProbes.length + }, + memory: { + collectedPhraseCount: getReusableWordstatEvidencePhraseSet(marketBeforeCollection.wordstat).size, + frontierProbeSeedCount, + frontierSourceTaskId, + requestedPhraseCount: requestedPhrases.size, + suppressedPhraseCount: suppressedPhrases.size + }, + selectedProbes, + warnings, + candidateSourceCounts, + rejectedSummary, + rejectedSamples: auditItems + .filter((item) => item.status !== "selected") + .sort((left, right) => right.score - left.score || left.phrase.localeCompare(right.phrase, "ru")) + .slice(0, 24), + nextActions: buildWordstatProbePreviewNextActions({ + blockers, + selectedCount: selectedProbes.length + }) + }; +} + export async function runMarketEnrichment(projectId: string): Promise { const analysis = await getLatestSemanticAnalysis(projectId); const projectOntologyVersion = mapMarketOntologyVersion(await getBoundOntologyVersion(projectId, analysis)); const providers = await buildProviderRegistry(); const anchorReview = await getAnchorReviewContract(projectId); - const normalizedSeeds: WordstatSeedCandidate[] = anchorReview.wordstatQueue - .map((seed) => ({ - clusterId: seed.clusterId, - clusterTitle: seed.clusterTitle, - confidence: seed.confidence, - phrase: seed.phrase, - priority: seed.priority, - reason: seed.reason, - source: seed.source - })); + const commercialDemand = await getSeoCommercialDemandContract(projectId); + const suppressedPhrases = await getSuppressedMarketPhraseSet( + projectId, + analysis?.runId ?? null, + projectOntologyVersion?.id ?? null + ); + const requestedPhrases = analysis ? await getRequestedWordstatPhraseSet(projectId, analysis.runId) : new Set(); + const blockedProbePhrases = new Set([...requestedPhrases, ...suppressedPhrases]); + const anchorProbeSeeds = buildAnchorWordstatProbeSeeds(anchorReview, suppressedPhrases); if ( !analysis || !isOntologyReadyForMarket(projectOntologyVersion) || anchorReview.state !== "approved" || getProviderStatus(providers, "wordstat") !== "connected" || - normalizedSeeds.length === 0 + anchorProbeSeeds.length === 0 ) { return getMarketEnrichmentContract(projectId); } try { - await collectWordstatEvidence(projectId, analysis, normalizedSeeds); - const marketAfterBaseCollection = await getMarketEnrichmentContract(projectId); - const expansionSeeds = buildWordstatExpansionSeeds(analysis, marketAfterBaseCollection); + const marketBeforeCollection = await getMarketEnrichmentContract(projectId); - if (expansionSeeds.length > 0) { - await collectWordstatEvidence(projectId, analysis, [...normalizedSeeds, ...expansionSeeds]); + if (anchorReview.demandCollectionProfile !== "market_wide") { + const contextualProbeSeeds = selectContextualWordstatProbeSeeds( + anchorProbeSeeds, + marketBeforeCollection.wordstat, + blockedProbePhrases + ); + + if (contextualProbeSeeds.length === 0) { + return marketBeforeCollection; + } + + await collectWordstatEvidenceBatches( + projectId, + analysis, + contextualProbeSeeds, + DEMAND_COLLECTION_MAX_BASE_RUNS, + DEMAND_COLLECTION_CONTEXTUAL_PROBE_BUDGET + ); + return getMarketEnrichmentContract(projectId); } + + const marketWidePlan = await buildMarketWideWordstatProbeCandidates({ + analysis, + anchorProbeSeeds, + anchorReview, + commercialDemand, + marketBeforeCollection, + projectId + }); + const frontierSeeds = selectMarketWideWordstatProbeSeeds( + marketWidePlan.candidates, + marketBeforeCollection.wordstat, + blockedProbePhrases + ); + + if (frontierSeeds.length === 0) { + return marketBeforeCollection; + } + + await collectWordstatEvidenceBatches( + projectId, + analysis, + frontierSeeds, + DEMAND_COLLECTION_MAX_MARKET_WIDE_RUNS, + DEMAND_COLLECTION_MARKET_WIDE_PROBE_BUDGET + ); } catch (error) { console.error(error); } diff --git a/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts b/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts new file mode 100644 index 0000000..eaf8176 --- /dev/null +++ b/seo_mode/seo_mode/server/src/market/marketFrontierDiscovery.ts @@ -0,0 +1,1206 @@ +import crypto from "node:crypto"; +import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js"; +import { getSeoCommercialDemandContract, type SeoCommercialDemandContract } from "../commercial/seoCommercialDemand.js"; +import { pool } from "../db/client.js"; +import { getAnchorReviewContract, type AnchorReviewContract } from "../keywords/anchorReview.js"; +import { getLatestWordstatEvidence, type WordstatEvidence, type WordstatSeedCandidate } from "./wordstatRepository.js"; + +type ProviderMode = "codex_manual" | "codex_workspace" | "deterministic_fallback"; +type FrontierType = + | "adjacent" + | "alternative_language" + | "commercial" + | "core" + | "implementation" + | "problem" + | "vertical"; +type FrontierRiskLevel = "adjacent" | "core" | "plausible" | "risky"; +type FrontierStatus = "covered" | "open" | "rejected" | "testing"; + +type MarketFrontierDiscoveryRunRow = { + id: string; + status: string; + input: Record; + output: MarketFrontierDiscoveryContract | null; + error_message: string | null; + started_at: Date | null; + completed_at: Date | null; + created_at: Date; +}; + +type MarketMemoryRows = { + requestedPhrases: string[]; + suppressedPhrases: string[]; + userAddedPhrases: string[]; +}; + +export type MarketFrontierProbeSeed = { + phrase: string; + normalizedPhrase: string; + frontierId: string; + frontierTitle: string; + frontierType: FrontierType; + clusterId: string; + clusterTitle: string; + priority: "high" | "medium" | "low"; + riskLevel: FrontierRiskLevel; + confidence: number; + reason: string; + expectedSignal: string; + dedupeAgainst: string[]; + evidenceRefs: string[]; +}; + +export type QueryLanguageFingerprint = { + schemaVersion: "query-language-fingerprint.v1"; + generatedFrom: { + exactPhraseCount: number; + relatedPhraseCount: number; + seedPhraseCount: number; + }; + workingModifiers: Array<{ count: number; examples: string[]; modifier: string }>; + workingPatterns: Array<{ count: number; examples: string[]; pattern: string }>; + marketNgrams: Array<{ count: number; examples: string[]; ngram: string }>; + noisyPatterns: Array<{ examples: string[]; pattern: string; reason: string }>; + summary: string; +}; + +export type MarketResearchMemory = { + schemaVersion: "market-research-memory.v1"; + requestedSeedCount: number; + requestedSeeds: string[]; + exactDemandCount: number; + exactDemandPhrases: string[]; + relatedDemandCount: number; + relatedDemandPhrases: string[]; + suppressedCount: number; + suppressedPhrases: string[]; + userAddedSignalCount: number; + userAddedSignals: string[]; + coveredPhrases: string[]; + openHypotheses: string[]; + summary: string; +}; + +export type MarketFrontierDiscoveryModelTaskContract = { + taskType: "seo.market_frontier_discovery"; + schemaVersion: "seo-market-frontier-discovery-task.v1"; + taskId: string; + projectId: string; + semanticRunId: string; + projectOntologyVersionId: string | null; + providerMode: "model_provider_contract"; + input: { + productUnderstanding: { + siteTopTerms: string[]; + semanticClusters: Array<{ + id: string; + title: string; + priority: SemanticAnalysisRun["clusters"][number]["priority"]; + targetIntent: string; + matchedTerms: string[]; + queryExamples: string[]; + }>; + commercialDemand: { + summary: string | null; + coreBuyerProblem: string | null; + coreCommercialValue: string | null; + coreDifferentiator: string | null; + buyerIntentFamilies: Array<{ + title: string; + buyerIntent: string; + priority: "high" | "medium" | "low"; + commercialAngle: string; + coreQualifier: string; + queryPatterns: string[]; + }>; + applicationCommercialAngles: Array<{ + sourceVector: string; + priority: "high" | "medium" | "low"; + commercialReframe: string; + coreQualifier: string; + queryPatterns: string[]; + }>; + }; + }; + approvedSemanticBasis: Array<{ + phrase: string; + clusterId: string; + clusterTitle: string; + intentType: string; + marketRole: string; + priority: "high" | "medium" | "low"; + normalizedFrom: string[]; + }>; + marketMemory: MarketResearchMemory; + queryLanguageFingerprint: QueryLanguageFingerprint; + }; + allowedActions: Array< + | "detect_covered_demand" + | "discover_market_frontiers" + | "preserve_product_fit" + | "propose_wordstat_probe_seeds" + | "reject_repeated_or_suppressed_queries" + | "use_query_language_fingerprint" + >; + outputSchema: { + schemaVersion: "market-frontier-discovery.v1"; + requiredTopLevelKeys: string[]; + probeSeedRequiredKeys: string[]; + }; + stopConditions: string[]; +}; + +export type MarketFrontierDiscoveryContract = { + schemaVersion: "market-frontier-discovery.v1"; + projectId: string; + semanticRunId: string | null; + projectOntologyVersionId: string | null; + generatedAt: string; + provider: { + mode: ProviderMode; + modelRequired: boolean; + message: string; + }; + sourceTaskId: string | null; + analystReview: { + status: "approved" | "needs_changes" | "not_reviewed" | "rejected"; + reviewer: "codex" | "human" | "system"; + reviewedAt: string | null; + notes: string[]; + }; + marketMemory: MarketResearchMemory; + queryLanguageFingerprint: QueryLanguageFingerprint; + frontiers: Array<{ + id: string; + title: string; + frontierType: FrontierType; + status: FrontierStatus; + riskLevel: FrontierRiskLevel; + rationale: string; + knownPhrases: string[]; + missingSignals: string[]; + probeSeeds: string[]; + evidenceRefs: string[]; + }>; + probeSeeds: MarketFrontierProbeSeed[]; + rejectedPatterns: Array<{ + pattern: string; + reason: string; + examples: string[]; + }>; + modelTask: MarketFrontierDiscoveryModelTaskContract | null; + summary: string; + nextActions: string[]; +}; + +const FRONTIER_RUN_TYPE = "market_frontier_discovery"; +const GENERIC_QUERY_TERMS = new Set([ + "ai", + "ii", + "ии", + "для", + "как", + "что", + "это", + "без", + "под", + "при", + "или", + "выбор", + "система", + "системы", + "платформа", + "платформы", + "решение", + "решения", + "сервис", + "бизнес", + "процесс", + "процессы", + "задач", + "задачи" +]); +const WORKING_MODIFIER_PATTERNS: Array<[RegExp, string]> = [ + [/(^|\s)автоматизац[а-яёa-z0-9-]*(\s|$)/i, "автоматизация"], + [/(^|\s)внедрени[ея](\s|$)/i, "внедрение"], + [/(^|\s)разработк[аи](\s|$)/i, "разработка"], + [/(^|\s)создани[ея](\s|$)/i, "создание"], + [/(^|\s)применени[ея](\s|$)/i, "применение"], + [/(^|\s)интеграц[ияеи](\s|$)/i, "интеграция"], + [/(^|\s)(купить|покупк[аи]|заказать|заказ)(\s|$)/i, "покупка/заказ"], + [/(^|\s)(стоимость|цена|тариф)(\s|$)/i, "стоимость"], + [/для\s+[а-яёa-z0-9-]+/i, "для X"], + [/\sв\s+[а-яёa-z0-9-]+/i, "в отрасли"] +]; + +function normalizePhrase(value: string) { + return value.trim().replace(/\s+/g, " ").toLocaleLowerCase("ru-RU"); +} + +function wordCount(value: string) { + return normalizePhrase(value).split(/\s+/).filter(Boolean).length; +} + +function unique(values: string[]) { + const seen = new Set(); + const result: string[] = []; + + for (const value of values) { + const normalizedValue = normalizePhrase(value); + + if (!normalizedValue || seen.has(normalizedValue)) { + continue; + } + + seen.add(normalizedValue); + result.push(normalizedValue); + } + + return result; +} + +function toId(value: string) { + return normalizePhrase(value) + .replace(/[^a-zа-яё0-9]+/gi, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80); +} + +function clampConfidence(value: number) { + return Math.max(0.1, Math.min(0.95, Number.isFinite(value) ? value : 0.55)); +} + +function getMarketFrontierMemoryHash(input: { + fingerprint: QueryLanguageFingerprint; + memory: MarketResearchMemory; + projectOntologyVersionId: string | null; + semanticRunId: string; +}) { + return crypto + .createHash("sha256") + .update( + JSON.stringify({ + semanticRunId: input.semanticRunId, + projectOntologyVersionId: input.projectOntologyVersionId, + requestedSeeds: input.memory.requestedSeeds, + coveredPhrases: input.memory.coveredPhrases, + suppressedPhrases: input.memory.suppressedPhrases, + userAddedSignals: input.memory.userAddedSignals, + openHypotheses: input.memory.openHypotheses, + fingerprint: { + generatedFrom: input.fingerprint.generatedFrom, + marketNgrams: input.fingerprint.marketNgrams.slice(0, 18), + workingModifiers: input.fingerprint.workingModifiers.slice(0, 12), + workingPatterns: input.fingerprint.workingPatterns.slice(0, 10) + } + }) + ) + .digest("hex") + .slice(0, 12); +} + +function isPlausibleProbePhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + return ( + wordCount(normalizedPhrase) >= 2 && + wordCount(normalizedPhrase) <= 7 && + /[а-яё]/i.test(normalizedPhrase) && + !["n/a", "na", "null", "undefined", "нет данных", "не применимо"].includes(normalizedPhrase) && + !/[.!?:;,]/.test(normalizedPhrase) && + !/^(автоматизац[а-яёa-z0-9-]*|применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(выбор|оценк[аи]|ценност[ьи])$/i.test(normalizedPhrase) && + !/^(применени[ея]|внедрени[ея]|разработк[аи]|создани[ея])\s+(данн[а-яё]*|процесс[а-яё]*|задач[а-яё]*)$/i.test(normalizedPhrase) && + !/[{}[\]<>#$%^*=~`]/.test(normalizedPhrase) + ); +} + +function getDefaultAnalystReview(): MarketFrontierDiscoveryContract["analystReview"] { + return { + notes: [], + reviewedAt: null, + reviewer: "system", + status: "not_reviewed" + }; +} + +function getFrequency(result: { frequency: number | null }) { + return result.frequency ?? 0; +} + +function getSourcePhraseKind(result: { phrase: string; sourcePhrase: string | null }) { + return result.sourcePhrase && normalizePhrase(result.sourcePhrase) !== normalizePhrase(result.phrase) ? "related" : "exact"; +} + +function addExample(map: Map, key: string, example: string) { + const normalizedKey = normalizePhrase(key); + + if (!normalizedKey) { + return; + } + + const current = map.get(normalizedKey) ?? { count: 0, examples: [] }; + current.count += 1; + + if (current.examples.length < 4 && !current.examples.map(normalizePhrase).includes(normalizePhrase(example))) { + current.examples.push(example); + } + + map.set(normalizedKey, current); +} + +function getPhraseTokens(phrase: string) { + return normalizePhrase(phrase) + .replace(/ё/g, "е") + .split(/[^a-zа-я0-9]+/i) + .map((token) => token.trim()) + .filter((token) => token.length >= 3 && !GENERIC_QUERY_TERMS.has(token)); +} + +function getNgrams(tokens: string[], size: number) { + const ngrams: string[] = []; + + for (let index = 0; index <= tokens.length - size; index += 1) { + ngrams.push(tokens.slice(index, index + size).join(" ")); + } + + return ngrams; +} + +function getPatternForPhrase(phrase: string) { + const normalizedPhrase = normalizePhrase(phrase); + + if (/^(внедрени[ея]|разработк[аи]|создани[ея]|применени[ея]|интеграц[ияеи]|автоматизац[а-яёa-z0-9-]*)\s+/i.test(normalizedPhrase)) { + return `${normalizedPhrase.split(/\s+/)[0]} X`; + } + + if (/\sдля\s+/i.test(normalizedPhrase)) { + return "X для Y"; + } + + if (/\sв\s+/i.test(normalizedPhrase)) { + return "X в Y"; + } + + if (/^(ии|ai)\s+/i.test(normalizedPhrase)) { + return "ИИ + объект"; + } + + return wordCount(normalizedPhrase) <= 3 ? "короткий category probe" : "длинный уточняющий probe"; +} + +function buildQueryLanguageFingerprint(wordstat: WordstatEvidence, suppressedPhrases: string[]): QueryLanguageFingerprint { + const modifierMap = new Map(); + const patternMap = new Map(); + const ngramMap = new Map(); + const phrases = [ + ...wordstat.seedResults.map((result) => ({ + phrase: result.phrase, + sourcePhrase: result.sourcePhrase, + frequency: result.frequency + })), + ...wordstat.topResults.map((result) => ({ + phrase: result.phrase, + sourcePhrase: result.sourcePhrase, + frequency: result.frequency + })) + ] + .filter((result) => result.frequency !== null) + .sort((left, right) => getFrequency(right) - getFrequency(left)); + + for (const result of phrases) { + for (const [pattern, modifier] of WORKING_MODIFIER_PATTERNS) { + if (pattern.test(result.phrase)) { + addExample(modifierMap, modifier, result.phrase); + } + } + + addExample(patternMap, getPatternForPhrase(result.phrase), result.phrase); + + const tokens = getPhraseTokens(result.phrase); + + for (const ngram of [...getNgrams(tokens, 1), ...getNgrams(tokens, 2)]) { + addExample(ngramMap, ngram, result.phrase); + } + } + + const workingModifiers = [...modifierMap.entries()] + .sort((left, right) => right[1].count - left[1].count || left[0].localeCompare(right[0], "ru")) + .slice(0, 16) + .map(([modifier, value]) => ({ modifier, ...value })); + const workingPatterns = [...patternMap.entries()] + .sort((left, right) => right[1].count - left[1].count || left[0].localeCompare(right[0], "ru")) + .slice(0, 12) + .map(([pattern, value]) => ({ pattern, ...value })); + const marketNgrams = [...ngramMap.entries()] + .filter(([ngram]) => !GENERIC_QUERY_TERMS.has(ngram)) + .sort((left, right) => right[1].count - left[1].count || left[0].localeCompare(right[0], "ru")) + .slice(0, 24) + .map(([ngram, value]) => ({ ngram, ...value })); + + return { + schemaVersion: "query-language-fingerprint.v1", + generatedFrom: { + exactPhraseCount: wordstat.seedResults.filter((result) => result.frequency !== null).length, + relatedPhraseCount: wordstat.topResults.filter((result) => getSourcePhraseKind(result) === "related").length, + seedPhraseCount: wordstat.seedResults.length + }, + workingModifiers, + workingPatterns, + marketNgrams, + noisyPatterns: suppressedPhrases.slice(0, 18).map((phrase) => ({ + examples: [phrase], + pattern: getPatternForPhrase(phrase), + reason: "Пользователь или backend suppression запретил возвращать эту формулировку в Stage 3." + })), + summary: + phrases.length > 0 + ? `Слепок построен по ${phrases.length} частотным Wordstat-фразам: основные паттерны ${workingPatterns.slice(0, 3).map((item) => item.pattern).join(", ") || "не выделены"}.` + : "Wordstat-слепок пока пуст: первые market-wide probes должны собрать язык рынка." + }; +} + +async function getMarketMemoryRows( + projectId: string, + semanticRunId: string | null, + projectOntologyVersionId: string | null, + anchorReview: AnchorReviewContract +): Promise { + const [jobsResult, decisionsResult] = await Promise.all([ + semanticRunId + ? pool.query<{ requested_phrases: string[] | null }>( + ` + select requested_phrases + from wordstat_jobs + where project_id = $1 and semantic_run_id = $2 + order by created_at desc + limit 40; + `, + [projectId, semanticRunId] + ) + : Promise.resolve({ rows: [] }), + semanticRunId && projectOntologyVersionId + ? pool.query<{ normalized_phrase: string | null; phrase: string | null }>( + ` + select phrase, payload->>'normalizedPhrase' as normalized_phrase + from keyword_decisions + where project_id = $1 + and semantic_run_id = $2 + and project_ontology_version_id = $3 + and status = 'suppressed' + and (nullif(trim(phrase), '') is not null or nullif(trim(payload->>'normalizedPhrase'), '') is not null) + order by created_at desc + limit 400; + `, + [projectId, semanticRunId, projectOntologyVersionId] + ) + : Promise.resolve({ rows: [] }) + ]); + const requestedPhrases = unique( + jobsResult.rows.flatMap((row) => Array.isArray(row.requested_phrases) ? row.requested_phrases : []) + ); + const suppressedPhrases = unique( + decisionsResult.rows.flatMap((row) => [row.normalized_phrase, row.phrase].filter((phrase): phrase is string => Boolean(phrase))) + ); + const userAddedPhrases = unique( + anchorReview.wordstatQueue + .filter((anchor) => /ручн|manual|user/i.test(`${anchor.clusterTitle} ${anchor.reason}`)) + .map((anchor) => anchor.phrase) + ); + + return { + requestedPhrases, + suppressedPhrases, + userAddedPhrases + }; +} + +function buildMarketResearchMemory( + wordstat: WordstatEvidence, + rows: MarketMemoryRows, + anchorReview: AnchorReviewContract +): MarketResearchMemory { + const exactDemandPhrases = unique( + wordstat.seedResults + .filter((result) => result.frequency !== null) + .sort((left, right) => (right.frequency ?? 0) - (left.frequency ?? 0)) + .map((result) => result.phrase) + ).slice(0, 80); + const relatedDemandPhrases = unique( + wordstat.topResults + .filter((result) => result.frequency !== null) + .sort((left, right) => (right.frequency ?? 0) - (left.frequency ?? 0)) + .map((result) => result.phrase) + ).slice(0, 160); + const coveredPhrases = unique([...rows.requestedPhrases, ...exactDemandPhrases, ...relatedDemandPhrases]); + const openHypotheses = unique( + anchorReview.wordstatQueue + .filter((anchor) => !coveredPhrases.includes(normalizePhrase(anchor.phrase))) + .map((anchor) => anchor.phrase) + ).slice(0, 80); + + return { + schemaVersion: "market-research-memory.v1", + requestedSeedCount: rows.requestedPhrases.length, + requestedSeeds: rows.requestedPhrases.slice(0, 1_000), + exactDemandCount: exactDemandPhrases.length, + exactDemandPhrases, + relatedDemandCount: relatedDemandPhrases.length, + relatedDemandPhrases, + suppressedCount: rows.suppressedPhrases.length, + suppressedPhrases: rows.suppressedPhrases.slice(0, 1_000), + userAddedSignalCount: rows.userAddedPhrases.length, + userAddedSignals: rows.userAddedPhrases.slice(0, 300), + coveredPhrases: coveredPhrases.slice(0, 1_500), + openHypotheses, + summary: + `Память рынка: ${rows.requestedPhrases.length} Wordstat probes уже отправлено, ` + + `${exactDemandPhrases.length} exact-фраз и ${relatedDemandPhrases.length} top/related-фраз покрыто, ` + + `${rows.suppressedPhrases.length} suppressed запрещено к возврату.` + }; +} + +function inferFrontierType(value: string, buyerIntent?: string): FrontierType { + const text = normalizePhrase(`${value} ${buyerIntent ?? ""}`); + + if (/(промышлен|строител|бпла|робот|документооборот|учет|малый бизнес|медицина|логистик|ритейл|образован|финанс)/i.test(text)) { + return "vertical"; + } + + if (/(внедрен|разработ|создан|интеграц|запуск|пилот)/i.test(text)) { + return "implementation"; + } + + if (/(стоим|цена|купить|заказать|тариф|поставщик|подрядчик|выбор)/i.test(text)) { + return "commercial"; + } + + if (/(боль|проблем|задач|рутин|контрол|ошибк|ускор|снизить)/i.test(text)) { + return "problem"; + } + + if (/(как называют|альтернатив|синоним|категор)/i.test(text)) { + return "alternative_language"; + } + + return "core"; +} + +function getFrontierRiskLevel(frontierType: FrontierType, priority: "high" | "medium" | "low"): FrontierRiskLevel { + if (frontierType === "core" || frontierType === "commercial" || priority === "high") { + return "core"; + } + + if (frontierType === "vertical" || frontierType === "implementation" || frontierType === "problem") { + return "plausible"; + } + + if (frontierType === "adjacent") { + return "adjacent"; + } + + return "plausible"; +} + +function getCommercialSeedPhrases(commercialDemand: SeoCommercialDemandContract | null) { + if (!commercialDemand) { + return []; + } + + return [ + ...commercialDemand.buyerIntentFamilies.slice(0, 16).flatMap((family) => + unique([ + ...family.queryPatterns, + family.title, + family.commercialAngle, + `${family.coreQualifier} ${family.title}` + ]) + .filter(isPlausibleProbePhrase) + .map((phrase) => ({ + phrase, + title: family.title, + priority: family.priority, + sourceText: `${family.buyerIntent} ${family.commercialAngle}`, + evidenceRefs: [`commercialDemand.family:${family.id}`] + })) + ), + ...commercialDemand.applicationCommercialAngles.slice(0, 18).flatMap((angle) => + unique([ + ...angle.queryPatterns, + angle.sourceVector, + angle.commercialReframe, + `${angle.coreQualifier} ${angle.sourceVector}` + ]) + .filter(isPlausibleProbePhrase) + .map((phrase) => ({ + phrase, + title: angle.sourceVector, + priority: angle.priority, + sourceText: angle.commercialReframe, + evidenceRefs: [`commercialDemand.angle:${angle.id}`] + })) + ) + ]; +} + +function getAnchorSeedPhrases(anchorReview: AnchorReviewContract) { + return anchorReview.wordstatQueue.slice(0, 120).map((anchor) => ({ + phrase: anchor.phrase, + title: anchor.clusterTitle, + priority: anchor.priority, + sourceText: `${anchor.intentType} ${anchor.marketRole} ${anchor.reason}`, + clusterId: anchor.clusterId, + clusterTitle: anchor.clusterTitle, + confidence: anchor.confidence, + evidenceRefs: [`anchor:${anchor.anchorId}`] + })); +} + +function getFingerprintProbePhrases(fingerprint: QueryLanguageFingerprint, anchorReview: AnchorReviewContract) { + const anchorTerms = unique( + anchorReview.wordstatQueue + .flatMap((anchor) => [anchor.phrase, anchor.clusterTitle, ...anchor.normalizedFrom]) + .flatMap((value) => getPhraseTokens(value).slice(0, 3)) + ).slice(0, 16); + const modifiers = fingerprint.workingModifiers + .map((item) => item.modifier) + .filter((modifier) => !/^для x$|^в отрасли$/i.test(modifier)) + .slice(0, 8); + + return modifiers.flatMap((modifier) => + anchorTerms.slice(0, 6).map((term) => ({ + phrase: `${modifier} ${term}`, + title: `Языковой паттерн: ${modifier}`, + priority: "medium" as const, + sourceText: `Query language fingerprint связал рабочий модификатор "${modifier}" с продуктовым термином "${term}".`, + evidenceRefs: [`fingerprint.modifier:${modifier}`] + })) + ).filter((item) => isPlausibleProbePhrase(item.phrase)); +} + +function buildFallbackFrontiers(input: { + anchorReview: AnchorReviewContract; + commercialDemand: SeoCommercialDemandContract | null; + fingerprint: QueryLanguageFingerprint; + memory: MarketResearchMemory; +}) { + const covered = new Set(input.memory.coveredPhrases.map(normalizePhrase)); + const suppressed = new Set(input.memory.suppressedPhrases.map(normalizePhrase)); + const seedInputs = [ + ...getCommercialSeedPhrases(input.commercialDemand), + ...getAnchorSeedPhrases(input.anchorReview), + ...getFingerprintProbePhrases(input.fingerprint, input.anchorReview) + ]; + const grouped = new Map(); + + for (const seed of seedInputs) { + const normalizedPhrase = normalizePhrase(seed.phrase); + + if (!isPlausibleProbePhrase(normalizedPhrase) || suppressed.has(normalizedPhrase)) { + continue; + } + + const frontierType = inferFrontierType(`${seed.phrase} ${seed.sourceText}`); + const key = `${frontierType}:${toId(seed.title) || toId(seed.phrase)}`; + const current = grouped.get(key) ?? { + evidenceRefs: [], + frontierType, + knownPhrases: [], + priority: seed.priority, + probeSeeds: [], + rationale: seed.sourceText || `Проверить рыночную ветку "${seed.title}" через Wordstat probe.`, + riskLevel: getFrontierRiskLevel(frontierType, seed.priority), + title: seed.title + }; + + current.evidenceRefs = unique([...current.evidenceRefs, ...seed.evidenceRefs]); + current.knownPhrases = unique([...current.knownPhrases, seed.phrase, ...(covered.has(normalizedPhrase) ? [seed.phrase] : [])]).slice(0, 18); + + if (!covered.has(normalizedPhrase) && current.probeSeeds.length < 8) { + current.probeSeeds = unique([...current.probeSeeds, seed.phrase]).slice(0, 8); + } + + grouped.set(key, current); + } + + return [...grouped.entries()] + .map(([key, value]) => ({ + id: `frontier:${key}`, + title: value.title, + frontierType: value.frontierType, + status: value.probeSeeds.length > 0 ? "open" as const : "covered" as const, + riskLevel: value.riskLevel, + rationale: value.rationale, + knownPhrases: value.knownPhrases, + missingSignals: value.probeSeeds.length > 0 + ? ["Нужен Wordstat top/related harvest для проверки языка и частотности этой ветки."] + : [], + probeSeeds: value.probeSeeds, + evidenceRefs: value.evidenceRefs + })) + .sort((left, right) => { + const statusWeight = (item: { status: FrontierStatus }) => item.status === "open" ? 1 : 0; + const riskWeight = (item: { riskLevel: FrontierRiskLevel }) => + item.riskLevel === "core" ? 3 : item.riskLevel === "plausible" ? 2 : item.riskLevel === "adjacent" ? 1 : 0; + + return statusWeight(right) - statusWeight(left) || riskWeight(right) - riskWeight(left); + }) + .slice(0, 24); +} + +function buildProbeSeedsFromFrontiers(frontiers: MarketFrontierDiscoveryContract["frontiers"], memory: MarketResearchMemory) { + const covered = new Set(memory.coveredPhrases.map(normalizePhrase)); + const suppressed = new Set(memory.suppressedPhrases.map(normalizePhrase)); + const seen = new Set(); + const seeds: MarketFrontierProbeSeed[] = []; + + for (const frontier of frontiers) { + for (const phrase of frontier.probeSeeds) { + const normalizedPhrase = normalizePhrase(phrase); + + if ( + !isPlausibleProbePhrase(normalizedPhrase) || + covered.has(normalizedPhrase) || + suppressed.has(normalizedPhrase) || + seen.has(normalizedPhrase) + ) { + continue; + } + + seen.add(normalizedPhrase); + seeds.push({ + phrase: normalizedPhrase, + normalizedPhrase, + frontierId: frontier.id, + frontierTitle: frontier.title, + frontierType: frontier.frontierType, + clusterId: frontier.id, + clusterTitle: frontier.title, + priority: frontier.riskLevel === "core" ? "high" : frontier.riskLevel === "plausible" ? "medium" : "low", + riskLevel: frontier.riskLevel, + confidence: frontier.riskLevel === "core" ? 0.78 : frontier.riskLevel === "plausible" ? 0.64 : 0.48, + reason: `Market frontier probe: ${frontier.rationale}`, + expectedSignal: "Wordstat top/related должен показать реальные формулировки рынка и частотность вокруг этой ветки.", + dedupeAgainst: ["marketMemory.coveredPhrases", "marketMemory.suppressedPhrases"], + evidenceRefs: frontier.evidenceRefs + }); + } + } + + return seeds.slice(0, 48); +} + +function buildModelTaskContract(input: { + analysis: SemanticAnalysisRun; + anchorReview: AnchorReviewContract; + commercialDemand: SeoCommercialDemandContract | null; + fingerprint: QueryLanguageFingerprint; + memory: MarketResearchMemory; + projectId: string; + projectOntologyVersionId: string | null; +}): MarketFrontierDiscoveryModelTaskContract { + return { + taskType: "seo.market_frontier_discovery", + schemaVersion: "seo-market-frontier-discovery-task.v1", + taskId: `${input.analysis.runId}:market-frontier-discovery:${getMarketFrontierMemoryHash({ + fingerprint: input.fingerprint, + memory: input.memory, + projectOntologyVersionId: input.projectOntologyVersionId, + semanticRunId: input.analysis.runId + })}:v1`, + projectId: input.projectId, + semanticRunId: input.analysis.runId, + projectOntologyVersionId: input.projectOntologyVersionId, + providerMode: "model_provider_contract", + input: { + productUnderstanding: { + siteTopTerms: input.analysis.styleProfile.topTerms.slice(0, 24).map((term) => term.term), + semanticClusters: input.analysis.clusters.slice(0, 18).map((cluster) => ({ + id: cluster.id, + title: cluster.title, + priority: cluster.priority, + targetIntent: cluster.targetIntent, + matchedTerms: cluster.matchedTerms.slice(0, 10), + queryExamples: cluster.queryExamples.slice(0, 8) + })), + commercialDemand: { + summary: input.commercialDemand?.summary ?? null, + coreBuyerProblem: input.commercialDemand?.commercialCore.coreBuyerProblem ?? null, + coreCommercialValue: input.commercialDemand?.commercialCore.coreCommercialValue ?? null, + coreDifferentiator: input.commercialDemand?.commercialCore.coreDifferentiator ?? null, + buyerIntentFamilies: + input.commercialDemand?.buyerIntentFamilies.slice(0, 16).map((family) => ({ + title: family.title, + buyerIntent: family.buyerIntent, + priority: family.priority, + commercialAngle: family.commercialAngle, + coreQualifier: family.coreQualifier, + queryPatterns: family.queryPatterns.slice(0, 8) + })) ?? [], + applicationCommercialAngles: + input.commercialDemand?.applicationCommercialAngles.slice(0, 18).map((angle) => ({ + sourceVector: angle.sourceVector, + priority: angle.priority, + commercialReframe: angle.commercialReframe, + coreQualifier: angle.coreQualifier, + queryPatterns: angle.queryPatterns.slice(0, 8) + })) ?? [] + } + }, + approvedSemanticBasis: input.anchorReview.wordstatQueue.slice(0, 120).map((anchor) => ({ + phrase: anchor.phrase, + clusterId: anchor.clusterId, + clusterTitle: anchor.clusterTitle, + intentType: anchor.intentType, + marketRole: anchor.marketRole, + priority: anchor.priority, + normalizedFrom: anchor.normalizedFrom + })), + marketMemory: input.memory, + queryLanguageFingerprint: input.fingerprint + }, + allowedActions: [ + "detect_covered_demand", + "discover_market_frontiers", + "preserve_product_fit", + "propose_wordstat_probe_seeds", + "reject_repeated_or_suppressed_queries", + "use_query_language_fingerprint" + ], + outputSchema: { + schemaVersion: "market-frontier-discovery.v1", + requiredTopLevelKeys: [ + "schemaVersion", + "projectId", + "semanticRunId", + "marketMemory", + "queryLanguageFingerprint", + "frontiers", + "probeSeeds", + "rejectedPatterns", + "summary", + "nextActions" + ], + probeSeedRequiredKeys: [ + "phrase", + "frontierId", + "frontierType", + "priority", + "riskLevel", + "reason" + ] + }, + stopConditions: [ + "Пользователь не обязан знать Wordstat или рынок: missing frontiers должна искать модель, а не пользователь.", + "Не предлагать probe, который уже есть в marketMemory.coveredPhrases, requestedSeeds или suppressedPhrases.", + "Не генерировать десятки перестановок одной фразы; один probe должен открывать top/related/popular harvest.", + "Использовать queryLanguageFingerprint: следующие probes должны повторять рабочие языковые паттерны рынка, а не внутренние слова сайта.", + "Отделять plausible/adjacent/risky: широкие рынки можно проверять, но нельзя сразу считать их core SEO without evidence.", + "Не подшивать логику под конкретный проект или домен; выводить универсальные buyer/problem/implementation/vertical frontiers.", + "Вернуть короткий probeSeeds список с объяснением expectedSignal и dedupeAgainst." + ] + }; +} + +function buildNextActions(contract: Omit) { + const actions: string[] = []; + + if (!contract.semanticRunId) { + return ["Сначала нужен semantic analysis: frontier discovery не может строить рыночную память без контекста проекта."]; + } + + if (contract.provider.mode === "deterministic_fallback") { + actions.push("Прогнать seo.market_frontier_discovery через AI Workspace, чтобы модель переосмыслила незакрытые рыночные ветки до Wordstat."); + } + + if (contract.probeSeeds.length > 0) { + actions.push(`Использовать ${contract.probeSeeds.length} frontier probe seeds как короткий market-wide вход в Wordstat.`); + } else { + actions.push("Новых frontier probes нет: текущая память считает проверенные ветки покрытыми или suppressed."); + } + + actions.push("После Wordstat обновить query language fingerprint и снова классифицировать Stage 3."); + + return actions; +} + +function buildFallbackContract(input: { + analysis: SemanticAnalysisRun | null; + anchorReview: AnchorReviewContract; + commercialDemand: SeoCommercialDemandContract | null; + fingerprint: QueryLanguageFingerprint; + memory: MarketResearchMemory; + projectId: string; + projectOntologyVersionId: string | null; +}): MarketFrontierDiscoveryContract { + const modelTask = input.analysis + ? buildModelTaskContract({ + analysis: input.analysis, + anchorReview: input.anchorReview, + commercialDemand: input.commercialDemand, + fingerprint: input.fingerprint, + memory: input.memory, + projectId: input.projectId, + projectOntologyVersionId: input.projectOntologyVersionId + }) + : null; + const frontiers = input.analysis + ? buildFallbackFrontiers({ + anchorReview: input.anchorReview, + commercialDemand: input.commercialDemand, + fingerprint: input.fingerprint, + memory: input.memory + }) + : []; + const contractWithoutActions: Omit = { + schemaVersion: "market-frontier-discovery.v1", + projectId: input.projectId, + semanticRunId: input.analysis?.runId ?? null, + projectOntologyVersionId: input.projectOntologyVersionId, + generatedAt: new Date().toISOString(), + provider: { + mode: "deterministic_fallback", + modelRequired: true, + message: + "Market frontier discovery собран fallback-ом из памяти, commercial demand и Wordstat fingerprint. Для максимального охвата нужен AI Workspace pass перед Wordstat." + }, + sourceTaskId: modelTask?.taskId ?? null, + analystReview: getDefaultAnalystReview(), + marketMemory: input.memory, + queryLanguageFingerprint: input.fingerprint, + frontiers, + probeSeeds: buildProbeSeedsFromFrontiers(frontiers, input.memory), + rejectedPatterns: input.memory.suppressedPhrases.slice(0, 24).map((phrase) => ({ + pattern: phrase, + reason: "Suppressed пользователем или backend-аудитом; не возвращать в Wordstat queue и Stage 3.", + examples: [phrase] + })), + modelTask, + summary: + frontiers.length > 0 + ? `Найдено ${frontiers.length} market frontiers, ${buildProbeSeedsFromFrontiers(frontiers, input.memory).length} probes доступны для Wordstat.` + : "Market frontier discovery пока не готов: нет semantic context или нет открытых гипотез." + }; + + return { + ...contractWithoutActions, + nextActions: buildNextActions(contractWithoutActions) + }; +} + +function mapRun(row: MarketFrontierDiscoveryRunRow | null): MarketFrontierDiscoveryContract | null { + if (!row?.output) { + return null; + } + + return { + ...row.output, + analystReview: row.output.analystReview ?? getDefaultAnalystReview() + }; +} + +function normalizeProbeSeed( + seed: Partial, + frontierById: Map, + memory: MarketResearchMemory +): MarketFrontierProbeSeed | null { + const phrase = typeof seed.phrase === "string" ? normalizePhrase(seed.phrase) : ""; + const frontier = typeof seed.frontierId === "string" ? frontierById.get(seed.frontierId) : null; + + if (!isPlausibleProbePhrase(phrase) || memory.coveredPhrases.includes(phrase) || memory.suppressedPhrases.includes(phrase)) { + return null; + } + + return { + phrase, + normalizedPhrase: phrase, + frontierId: seed.frontierId ?? frontier?.id ?? "frontier:model", + frontierTitle: seed.frontierTitle ?? frontier?.title ?? "Model frontier", + frontierType: seed.frontierType ?? frontier?.frontierType ?? "adjacent", + clusterId: seed.clusterId ?? seed.frontierId ?? frontier?.id ?? "market-frontier", + clusterTitle: seed.clusterTitle ?? seed.frontierTitle ?? frontier?.title ?? "Market frontier", + priority: seed.priority === "high" || seed.priority === "low" ? seed.priority : "medium", + riskLevel: seed.riskLevel ?? frontier?.riskLevel ?? "plausible", + confidence: clampConfidence(seed.confidence ?? 0.62), + reason: seed.reason ?? frontier?.rationale ?? "Model proposed market frontier probe.", + expectedSignal: seed.expectedSignal ?? "Проверить Wordstat top/related/popular вокруг новой рыночной ветки.", + dedupeAgainst: Array.isArray(seed.dedupeAgainst) ? seed.dedupeAgainst : ["marketMemory"], + evidenceRefs: Array.isArray(seed.evidenceRefs) ? seed.evidenceRefs : frontier?.evidenceRefs ?? [] + }; +} + +function normalizeModelOutput( + projectId: string, + output: MarketFrontierDiscoveryContract, + fallback: MarketFrontierDiscoveryContract, + providerMode: ProviderMode, + providerMessage: string +): MarketFrontierDiscoveryContract { + if (output.schemaVersion !== "market-frontier-discovery.v1") { + throw new Error("Market frontier discovery output schemaVersion должен быть market-frontier-discovery.v1."); + } + + if (output.projectId !== projectId) { + throw new Error("Market frontier discovery output projectId не совпадает с проектом."); + } + + const frontiers = Array.isArray(output.frontiers) ? output.frontiers : []; + const frontierById = new Map(frontiers.map((frontier) => [frontier.id, frontier])); + const probeSeeds = (Array.isArray(output.probeSeeds) ? output.probeSeeds : []) + .map((seed) => normalizeProbeSeed(seed, frontierById, fallback.marketMemory)) + .filter((seed): seed is MarketFrontierProbeSeed => Boolean(seed)); + const contractWithoutActions: Omit = { + ...output, + analystReview: output.analystReview ?? getDefaultAnalystReview(), + generatedAt: new Date().toISOString(), + provider: { + mode: providerMode, + modelRequired: providerMode !== "codex_manual", + message: providerMessage + }, + sourceTaskId: fallback.sourceTaskId, + marketMemory: fallback.marketMemory, + queryLanguageFingerprint: output.queryLanguageFingerprint ?? fallback.queryLanguageFingerprint, + frontiers, + probeSeeds, + modelTask: fallback.modelTask, + rejectedPatterns: Array.isArray(output.rejectedPatterns) ? output.rejectedPatterns : fallback.rejectedPatterns, + summary: output.summary || `Market frontier discovery вернул ${frontiers.length} frontiers и ${probeSeeds.length} Wordstat probes.` + }; + + return { + ...contractWithoutActions, + nextActions: buildNextActions(contractWithoutActions) + }; +} + +async function buildCurrentFallback(projectId: string): Promise { + const analysis = await getLatestSemanticAnalysis(projectId); + const [anchorReview, commercialDemand, wordstat] = await Promise.all([ + getAnchorReviewContract(projectId), + getSeoCommercialDemandContract(projectId), + getLatestWordstatEvidence(projectId, analysis?.runId ?? null) + ]); + const projectOntologyVersionId = analysis?.projectOntology?.versionId ?? anchorReview.projectOntologyVersionId ?? null; + const memoryRows = await getMarketMemoryRows(projectId, analysis?.runId ?? null, projectOntologyVersionId, anchorReview); + const fingerprint = buildQueryLanguageFingerprint(wordstat, memoryRows.suppressedPhrases); + const memory = buildMarketResearchMemory(wordstat, memoryRows, anchorReview); + + return buildFallbackContract({ + analysis, + anchorReview, + commercialDemand, + fingerprint, + memory, + projectId, + projectOntologyVersionId + }); +} + +async function getLatestAlignedMarketFrontierDiscovery( + projectId: string, + semanticRunId: string | null, + sourceTaskId: string | null +) { + if (!semanticRunId || !sourceTaskId) { + return null; + } + + const result = await pool.query( + ` + select id, status, input, output, error_message, started_at, completed_at, created_at + from runs + where project_id = $1 + and run_type = '${FRONTIER_RUN_TYPE}' + and output->>'semanticRunId' = $2 + and output->>'sourceTaskId' = $3 + order by created_at desc + limit 1; + `, + [projectId, semanticRunId, sourceTaskId] + ); + + return mapRun(result.rows[0] ?? null); +} + +export async function getMarketFrontierDiscoveryContract(projectId: string): Promise { + const fallback = await buildCurrentFallback(projectId); + const latest = await getLatestAlignedMarketFrontierDiscovery(projectId, fallback.semanticRunId, fallback.sourceTaskId); + + return latest ? { + ...latest, + marketMemory: fallback.marketMemory, + modelTask: fallback.modelTask, + queryLanguageFingerprint: latest.queryLanguageFingerprint ?? fallback.queryLanguageFingerprint + } : fallback; +} + +export async function buildLatestMarketFrontierDiscoveryTask( + projectId: string +): Promise { + return (await buildCurrentFallback(projectId)).modelTask; +} + +export async function saveMarketFrontierDiscoveryFromModelProvider( + projectId: string, + output: MarketFrontierDiscoveryContract, + sourceModelTaskRunId: string +): Promise { + const fallback = await buildCurrentFallback(projectId); + const normalizedOutput = normalizeModelOutput( + projectId, + output, + fallback, + "codex_workspace", + "AI Workspace Codex provider выполнил seo.market_frontier_discovery: результат сохранён как рыночная память, fingerprint и probe plan перед Wordstat." + ); + const result = await pool.query( + ` + insert into runs (project_id, run_type, status, input, output, started_at, completed_at) + values ($1, '${FRONTIER_RUN_TYPE}', 'done', $2::jsonb, $3::jsonb, now(), now()) + returning id, status, input, output, error_message, started_at, completed_at, created_at; + `, + [ + projectId, + JSON.stringify({ + providerMode: "codex_workspace", + schemaVersion: "market-frontier-discovery-model-provider-input.v1", + semanticRunId: normalizedOutput.semanticRunId, + sourceModelTaskRunId, + sourceTaskId: normalizedOutput.sourceTaskId, + taskType: "seo.market_frontier_discovery" + }), + JSON.stringify(normalizedOutput) + ] + ); + const run = mapRun(result.rows[0] ?? null); + + if (!run) { + throw new Error("Не удалось сохранить AI Workspace market frontier discovery."); + } + + return run; +} + +export function mapMarketFrontierProbeSeedsToWordstatCandidates( + discovery: MarketFrontierDiscoveryContract +): WordstatSeedCandidate[] { + return discovery.probeSeeds.map((seed) => ({ + clusterId: seed.clusterId, + clusterTitle: seed.clusterTitle, + confidence: seed.confidence, + phrase: seed.phrase, + priority: seed.priority, + reason: seed.reason, + source: "detected" + })); +} diff --git a/seo_mode/seo_mode/server/src/market/wordstatProvider.ts b/seo_mode/seo_mode/server/src/market/wordstatProvider.ts index 084a4c9..028f56a 100644 --- a/seo_mode/seo_mode/server/src/market/wordstatProvider.ts +++ b/seo_mode/seo_mode/server/src/market/wordstatProvider.ts @@ -100,6 +100,20 @@ function getString(value: unknown) { return typeof value === "string" ? value.trim() : ""; } +function normalizePhrase(value: string) { + return value.trim().replace(/\s+/g, " ").toLowerCase(); +} + +function isRealWordstatPhrase(value: string) { + const normalizedValue = normalizePhrase(value); + + return ( + normalizedValue.length > 1 && + /[a-zа-яё0-9]/i.test(normalizedValue) && + !["-", "—", "n/a", "na", "null", "undefined", "нет данных", "не применимо"].includes(normalizedValue) + ); +} + function getNumber(value: unknown) { if (typeof value === "number" && Number.isFinite(value)) { return Math.max(0, Math.round(value)); @@ -124,6 +138,10 @@ function asArray(value: unknown) { return Array.isArray(value) ? value : []; } +function mergeArraySources(...values: unknown[]) { + return values.flatMap((value) => asArray(value)); +} + function normalizeRelatedQuery(value: unknown): WordstatRelatedQuery | null { const record = asRecord(value); const phrase = @@ -133,7 +151,7 @@ function normalizeRelatedQuery(value: unknown): WordstatRelatedQuery | null { getString(record.text) || getString(record.name); - if (!phrase) { + if (!phrase || !isRealWordstatPhrase(phrase)) { return null; } @@ -162,20 +180,27 @@ function normalizeSeedResult(value: unknown, seed: WordstatSeedInput): WordstatS getNumber(record.count) ?? getNumber(record.total) ?? getNumber(record.impressions); - const relatedSource = - record.relatedQueries ?? - record.related_queries ?? - record.queries ?? - record.items ?? - record.results ?? - record.associations; + const relatedSource = mergeArraySources( + record.relatedQueries, + record.related_queries, + record.queries, + record.items, + record.results, + record.associations, + record.topRequests, + record.top_requests, + record.top, + record.popular, + record.popularQueries, + record.popular_queries + ); return { seedId: seed.seedId, - seedPhrase: phrase, + seedPhrase: isRealWordstatPhrase(phrase) ? phrase : seed.phrase, frequency, frequencyGroup: getFrequencyGroup(frequency, record.frequencyGroup ?? record.frequency_group ?? record.group), - relatedQueries: asArray(relatedSource).flatMap((related) => { + relatedQueries: relatedSource.flatMap((related) => { const normalized = normalizeRelatedQuery(related); return normalized ? [normalized] : []; }), @@ -237,14 +262,22 @@ class DisabledWordstatProvider implements WordstatProvider { } async function postJson(endpoint: string, body: unknown, headers: Record = {}) { - const response = await fetch(endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json", - ...headers - }, - body: JSON.stringify(body) - }); + let response: Response; + try { + response = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...headers + }, + body: JSON.stringify(body) + }); + } catch (error) { + if (error instanceof TypeError && /fetch failed/i.test(error.message)) { + throw new Error(`Wordstat provider недоступен: backend не смог подключиться к ${endpoint}.`); + } + throw error; + } if (!response.ok) { const message = await response.text().catch(() => ""); diff --git a/seo_mode/seo_mode/server/src/market/wordstatRepository.ts b/seo_mode/seo_mode/server/src/market/wordstatRepository.ts index b9181a0..49e95bf 100644 --- a/seo_mode/seo_mode/server/src/market/wordstatRepository.ts +++ b/seo_mode/seo_mode/server/src/market/wordstatRepository.ts @@ -66,6 +66,7 @@ type WordstatResultRow = { const WORDSTAT_STATS_REQUESTS_PER_HOUR_LIMIT = 100; const WORDSTAT_STATS_REQUESTS_PER_SECOND_LIMIT = 10; +const WORDSTAT_MAX_SAFE_SEEDS_PER_COLLECTION = 16; export type WordstatQuotaSnapshot = { provider: string; @@ -154,8 +155,18 @@ function normalizePhrase(value: string) { return value.trim().replace(/\s+/g, " ").toLowerCase(); } -const WORDSTAT_TOP_RESULTS_LIMIT = 160; -const WORDSTAT_TOP_RESULTS_PER_SOURCE_LIMIT = 12; +function isRealWordstatPhrase(value: string) { + const normalizedValue = normalizePhrase(value); + + return ( + normalizedValue.length > 1 && + /[a-zа-яё0-9]/i.test(normalizedValue) && + !["-", "—", "n/a", "na", "null", "undefined", "нет данных", "не применимо"].includes(normalizedValue) + ); +} + +const WORDSTAT_TOP_RESULTS_LIMIT = 240; +const WORDSTAT_TOP_RESULTS_PER_SOURCE_LIMIT = 16; const RELATED_QUERY_STOP_WORDS = new Set([ "a", @@ -270,7 +281,7 @@ function shouldKeepRelatedQuery(seedPhrase: string, relatedPhrase: string) { const normalizedSeedPhrase = normalizePhrase(seedPhrase); const normalizedRelatedPhrase = normalizePhrase(relatedPhrase); - if (!normalizedRelatedPhrase || normalizedRelatedPhrase === normalizedSeedPhrase) { + if (!isRealWordstatPhrase(normalizedRelatedPhrase) || normalizedRelatedPhrase === normalizedSeedPhrase) { return false; } @@ -352,7 +363,7 @@ function getRepresentativeTopResultRows(results: WordstatResultRow[]) { const topResultByPhrase = new Map(); for (const result of results) { - if (result.frequency === null) { + if (result.frequency === null || !isRealWordstatPhrase(result.phrase)) { continue; } @@ -448,10 +459,11 @@ function buildEvidence( results: WordstatResultRow[], quota: WordstatQuotaSnapshot | null ): WordstatEvidence { - const seedRows = results.filter((result) => result.source_phrase === result.phrase || !result.source_phrase); + const realResults = results.filter((result) => isRealWordstatPhrase(result.phrase)); + const seedRows = realResults.filter((result) => result.source_phrase === result.phrase || !result.source_phrase); const relatedCounts = new Map(); - for (const result of results) { + for (const result of realResults) { if (result.source_phrase && result.source_phrase !== result.phrase) { relatedCounts.set(result.source_phrase, (relatedCounts.get(result.source_phrase) ?? 0) + 1); } @@ -475,7 +487,7 @@ function buildEvidence( seedSource: result.seed_source, collectedAt: result.created_at.toISOString() })), - topResults: getRepresentativeTopResultRows(results) + topResults: getRepresentativeTopResultRows(realResults) .map((result) => ({ id: result.id, phrase: result.phrase, @@ -497,11 +509,11 @@ function buildEvidence( noSignalSeedCount: seedRows.filter( (result) => result.frequency === null && (relatedCounts.get(result.phrase) ?? 0) === 0 ).length, - resultCount: results.length, - totalFrequency: results.reduce((sum, result) => sum + (result.frequency ?? 0), 0), - highFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "high").length, - midFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "mid").length, - lowFrequencyCount: results.filter((result) => getFrequencyBucket(result.frequency_group) === "low").length + resultCount: realResults.length, + totalFrequency: realResults.reduce((sum, result) => sum + (result.frequency ?? 0), 0), + highFrequencyCount: realResults.filter((result) => getFrequencyBucket(result.frequency_group) === "high").length, + midFrequencyCount: realResults.filter((result) => getFrequencyBucket(result.frequency_group) === "mid").length, + lowFrequencyCount: realResults.filter((result) => getFrequencyBucket(result.frequency_group) === "low").length } }; } @@ -599,6 +611,27 @@ async function getCollectedSeedPhraseSet(projectId: string, semanticRunId: strin return new Set(result.rows.map((row) => normalizePhrase(row.normalized_phrase ?? row.phrase))); } +export async function getRequestedWordstatPhraseSet(projectId: string, semanticRunId: string) { + const result = await pool.query<{ requested_phrases: string[] | null }>( + ` + select requested_phrases + from wordstat_jobs + where project_id = $1 + and semantic_run_id = $2 + and status in ('queued', 'running', 'done', 'failed') + and jsonb_array_length(requested_phrases) > 0; + `, + [projectId, semanticRunId] + ); + + return new Set( + result.rows + .flatMap((row) => Array.isArray(row.requested_phrases) ? row.requested_phrases : []) + .map(normalizePhrase) + .filter(Boolean) + ); +} + async function getWordstatQuotaSnapshot(provider: string, mode: string): Promise { const result = await pool.query<{ id: string; @@ -860,7 +893,8 @@ async function updateJobFailed(jobId: string, error: unknown) { export async function collectWordstatEvidence( projectId: string, analysis: SemanticAnalysisRun, - seedCandidates: WordstatSeedCandidate[] = analysis.seedCandidates + seedCandidates: WordstatSeedCandidate[] = analysis.seedCandidates, + options: { maxSeeds?: number } = {} ) { const provider = await createWordstatProvider(); const status = provider.getStatus(); @@ -871,14 +905,20 @@ export async function collectWordstatEvidence( const seedRows = await ensureSeedRows(projectId, analysis, seedCandidates); const collectedSeedPhrases = await getCollectedSeedPhraseSet(projectId, analysis.runId); - const uncollectedSeedRows = seedRows.filter((row) => !collectedSeedPhrases.has(normalizePhrase(row.phrase))); + const requestedSeedPhrases = await getRequestedWordstatPhraseSet(projectId, analysis.runId); + const alreadyTriedSeedPhrases = new Set([...collectedSeedPhrases, ...requestedSeedPhrases]); + const uncollectedSeedRows = seedRows.filter((row) => !alreadyTriedSeedPhrases.has(normalizePhrase(row.phrase))); if (uncollectedSeedRows.length === 0) { return getLatestWordstatEvidence(projectId, analysis.runId); } const budget = await getWordstatHourlyBudget(status.provider, status.mode); - const maxSeedsForRun = Math.min(status.maxSeedsPerRun, budget.remaining); + const requestedMaxSeeds = Math.min( + options.maxSeeds ?? status.maxSeedsPerRun, + WORDSTAT_MAX_SAFE_SEEDS_PER_COLLECTION + ); + const maxSeedsForRun = Math.min(status.maxSeedsPerRun, budget.remaining, requestedMaxSeeds); if (maxSeedsForRun <= 0) { throw new Error( @@ -943,9 +983,7 @@ export async function getLatestWordstatEvidence( updated_at from wordstat_jobs where project_id = $1 and semantic_run_id = $2 - order by - case when status = 'done' then 0 else 1 end, - created_at desc + order by created_at desc limit 1; `, [projectId, semanticRunId] diff --git a/seo_mode/seo_mode/server/src/modelContext/seoModelContext.ts b/seo_mode/seo_mode/server/src/modelContext/seoModelContext.ts index d903ff4..131bcd1 100644 --- a/seo_mode/seo_mode/server/src/modelContext/seoModelContext.ts +++ b/seo_mode/seo_mode/server/src/modelContext/seoModelContext.ts @@ -1,8 +1,22 @@ import { getLatestSemanticAnalysis } from "../analysis/semanticAnalysis.js"; +import { + buildSeoBusinessSynthesisTask, + getLatestAlignedSeoBusinessSynthesis, + type SeoBusinessSynthesisModelTaskContract +} from "../business/seoBusinessSynthesis.js"; +import { + buildSeoCommercialDemandTask, + getLatestAlignedSeoCommercialDemand, + type SeoCommercialDemandModelTaskContract +} from "../commercial/seoCommercialDemand.js"; import { getLatestAlignedSeoContextReview } from "../contextReview/seoContextReview.js"; import { buildSerpInterpretationModelTask, type SerpInterpretationModelTaskContract } from "../evidence/serpInterpretationTask.js"; import { getLatestYandexEvidenceContract } from "../evidence/yandexEvidence.js"; import { getAnchorReviewContract } from "../keywords/anchorReview.js"; +import { + getMarketFrontierDiscoveryContract, + type MarketFrontierDiscoveryModelTaskContract +} from "../market/marketFrontierDiscovery.js"; import { getSeoModelProviderCatalog, type SeoModelProviderCatalog @@ -26,7 +40,10 @@ import { export type SeoModelTaskContract = | KeywordCleaningModelTaskContract + | SeoBusinessSynthesisModelTaskContract + | SeoCommercialDemandModelTaskContract | SeoContextReviewModelTaskContract + | MarketFrontierDiscoveryModelTaskContract | SeoNormalizationModelTaskContract | SerpInterpretationModelTaskContract | StrategyQualityReviewModelTaskContract @@ -64,8 +81,13 @@ export type SeoModelContextContract = { } | null; stageReadiness: { semanticContextReady: boolean; + businessSynthesisTaskReady: boolean; + businessSynthesisResultReady: boolean; + commercialDemandTaskReady: boolean; + commercialDemandResultReady: boolean; contextReviewTaskReady: boolean; contextReviewResultReady: boolean; + marketFrontierDiscoveryTaskReady: boolean; normalizationState: "not_ready" | "ready"; normalizationTaskReady: boolean; keywordCleaningState: "not_ready" | "ready"; @@ -80,7 +102,7 @@ export type SeoModelContextContract = { modelProvider: SeoModelProviderCatalog; contextReview: { runId: string; - providerMode: "codex_manual" | "deterministic_fallback"; + providerMode: "codex_manual" | "codex_workspace" | "deterministic_fallback"; generatedAt: string; needsHumanReview: boolean; summary: string; @@ -167,7 +189,12 @@ export async function getSeoModelContextContract(projectId: string): Promise( throw new Error("ai_workspace_assistant_timeout"); } + if (error instanceof TypeError && /fetch failed/i.test(error.message)) { + throw new Error("AI Workspace bridge недоступен: backend не смог подключиться к Assistant/Hub endpoint."); + } + throw error; } finally { clearTimeout(timeout); @@ -169,6 +177,14 @@ function buildSeoModelTaskPrompt(input: { `Required top-level keys: ${input.task.outputSchema.requiredTopLevelKeys.join(", ")}.`, "If the output contract has bookkeeping keys such as modelTask, sourceTaskId, source, semanticRunId, projectId, or provider, populate them from the task contract and the provided evidence.", "For keyword tasks, classify only phrases present in task.input.seedQueue or task.input.wordstatTopResults; keep unresolved or weak rows out of approved rewrite lanes.", + "For business synthesis tasks, behave like a senior product SEO strategist: read the site evidence, identify the central product offer, platform layers, application vectors, and demand families, while clearly separating explicit claims, conservative inferences, and risky expansion.", + "For commercial demand tasks, behave like a buyer-intent SEO strategist: convert product architecture into purchasing, evaluation, implementation, automation, integration, replacement, pilot, or vendor-selection demand. Preserve the product's core differentiator from productFrame/corePillars in every vertical/application query pattern; do not output pure informational or architecture-inventory phrases as demand.", + "For market frontier discovery tasks, behave like a market-research SEO strategist for a non-SEO user: read productUnderstanding, approvedSemanticBasis, marketMemory, and queryLanguageFingerprint; identify demand frontiers that current coveredPhrases/requestedSeeds do not cover; propose a short, high-leverage Wordstat probeSeeds list that can harvest top/related/popular rows. Do not return repeated, suppressed, n/a, or final Stage 3 keyword clutter.", + "For context review and normalization tasks, treat task.input.siteTextEvidence.chunks as the primary cleaned text corpus of the selected site scope when it is present.", + "For normalization tasks, use task.input.siteTextEvidence.highSignalLines as a checklist of short visible meanings: represent non-technical product, module, use-case, integration, audience, workflow, or vertical lines as needs-human query/corePhrase candidates, or put them in excludedPhrases with an explicit reason.", + "For normalization tasks, use task.input.businessSynthesis as the product hierarchy: keep productFrame/corePillars as the semantic center and keep applicationVectors as use cases or vertical branches unless the synthesis explicitly marks them core.", + "For normalization tasks, use task.input.commercialDemand as the buyer-intent frame: seedStrategy and wordstatQueries should be commercial/problem-solution/evaluation/implementation phrases, not raw architecture theses. seedStrategy must include the actual semantic-basis query candidates across buyer-intent groups, not only 3-4 core phrases; target 24-60 needs-human seeds and keep them concise enough for market review. Do not shorten by deleting the product core differentiator; a longer qualified buyer query is better than a short generic phrase for another market. If a phrase is only informational, put it in excludedPhrases or rejectedSeeds with a reason.", + "Decompose meanings from that corpus first, then use semanticSignals/pageSignals as backend hints and cross-checks.", "", "Hard boundaries:", "- Work only from the task contract below.", @@ -381,11 +397,52 @@ function parseJsonFromText(rawText: string): JsonRecord | null { } const firstBrace = text.indexOf("{"); - const lastBrace = text.lastIndexOf("}"); - if (firstBrace >= 0 && lastBrace > firstBrace) { + if (firstBrace >= 0) { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = firstBrace; index < text.length; index += 1) { + const char = text[index]; + + if (escaped) { + escaped = false; + continue; + } + + if (char === "\\") { + escaped = inString; + continue; + } + + if (char === "\"") { + inString = !inString; + continue; + } + + if (inString) { + continue; + } + + if (char === "{") { + depth += 1; + } + + if (char === "}") { + depth -= 1; + + if (depth === 0) { + try { + const parsed = JSON.parse(text.slice(firstBrace, index + 1)); + return isJsonRecord(parsed) ? parsed : null; + } catch {} + } + } + } + try { - const parsed = JSON.parse(text.slice(firstBrace, lastBrace + 1)); + const parsed = JSON.parse(text.slice(firstBrace)); return isJsonRecord(parsed) ? parsed : null; } catch {} } diff --git a/seo_mode/seo_mode/server/src/modelProvider/modelProviderRegistry.ts b/seo_mode/seo_mode/server/src/modelProvider/modelProviderRegistry.ts index f01af8e..4100a2a 100644 --- a/seo_mode/seo_mode/server/src/modelProvider/modelProviderRegistry.ts +++ b/seo_mode/seo_mode/server/src/modelProvider/modelProviderRegistry.ts @@ -3,6 +3,9 @@ import { getSeoAiWorkspaceConfigStatus } from "./aiWorkspaceBridgeConfig.js"; export const SEO_MODEL_TASK_TYPES = [ "seo.context_review", + "seo.business_synthesis", + "seo.commercial_demand", + "seo.market_frontier_discovery", "seo.normalization", "seo.keyword_cleaning", "seo.serp_interpretation", diff --git a/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts b/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts index 631df12..614b678 100644 --- a/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts +++ b/seo_mode/seo_mode/server/src/modelProvider/seoModelTaskRuns.ts @@ -1,7 +1,28 @@ import { pool } from "../db/client.js"; +import { + buildSeoBusinessSynthesisPromotionFallback, + saveSeoBusinessSynthesisFromModelProvider, + type SeoBusinessSynthesisContract, + type SeoBusinessSynthesisModelTaskContract +} from "../business/seoBusinessSynthesis.js"; +import { + buildSeoCommercialDemandPromotionFallback, + saveSeoCommercialDemandFromModelProvider, + type SeoCommercialDemandContract, + type SeoCommercialDemandModelTaskContract +} from "../commercial/seoCommercialDemand.js"; +import { saveSeoContextReviewFromModelProvider, type SeoContextReviewOutput } from "../contextReview/seoContextReview.js"; import { saveKeywordCleaningFromModelProvider, type KeywordCleaningContract } from "../keywords/keywordCleaning.js"; import { getSeoModelContextContract, type SeoModelTaskContract } from "../modelContext/seoModelContext.js"; -import { saveSeoNormalizationFromModelProvider, type SeoNormalizationContract } from "../normalization/seoNormalization.js"; +import { + saveMarketFrontierDiscoveryFromModelProvider, + type MarketFrontierDiscoveryContract +} from "../market/marketFrontierDiscovery.js"; +import { + saveSeoNormalizationFromModelProvider, + type SeoNormalizationContract, + type SeoNormalizationModelTaskContract +} from "../normalization/seoNormalization.js"; import { saveStrategyQualityReviewFromModelProvider, type StrategyQualityReviewContract } from "../strategy/strategyQualityReview.js"; import { saveStrategySynthesisFromModelProvider, type StrategySynthesisContract } from "../strategy/strategySynthesis.js"; import { @@ -157,6 +178,22 @@ function getOutputTokenBudget(task: SeoModelTaskContract | null) { return 2600; } + if (task.taskType === "seo.business_synthesis") { + return 4800; + } + + if (task.taskType === "seo.commercial_demand") { + return 5200; + } + + if (task.taskType === "seo.market_frontier_discovery") { + return 6200; + } + + if (task.taskType === "seo.normalization") { + return 4200; + } + if (task.taskType === "seo.keyword_cleaning") { return 9000; } @@ -417,6 +454,68 @@ function isRecord(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); } +function getRecordString(record: Record, key: string) { + const value = record[key]; + + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +async function getContextReviewPromotionFallback(projectId: string, run: SeoModelTaskRun) { + const inputTask = isRecord(run.input.task) ? run.input.task : {}; + const semanticRunId = getRecordString(inputTask, "semanticRunId") ?? run.task.semanticRunId; + const sourceTaskId = getRecordString(inputTask, "taskId") ?? run.task.taskId; + + if (!semanticRunId || !sourceTaskId) { + throw new Error("Context Review promotion fallback не может определить semanticRunId/sourceTaskId."); + } + + const analysisResult = await pool.query<{ output: { scanVersionId?: unknown; projectOntology?: { versionId?: unknown } } | null }>( + ` + select output + from runs + where id = $1 + and project_id = $2 + and run_type = 'semantic_analysis' + limit 1; + `, + [semanticRunId, projectId] + ); + const analysisOutput = analysisResult.rows[0]?.output ?? null; + const scanVersionId = getRecordString(inputTask, "scanVersionId") ?? ( + typeof analysisOutput?.scanVersionId === "string" ? analysisOutput.scanVersionId : null + ); + const projectOntologyVersionId = getRecordString(inputTask, "projectOntologyVersionId") ?? ( + typeof analysisOutput?.projectOntology?.versionId === "string" ? analysisOutput.projectOntology.versionId : null + ); + + if (!scanVersionId) { + throw new Error("Context Review promotion fallback не может определить scanVersionId."); + } + + return { + projectOntologyVersionId, + scanVersionId, + semanticRunId, + sourceTaskId + }; +} + +function getNormalizationPromotionFallback(run: SeoModelTaskRun) { + const inputTask = isRecord(run.input.task) ? run.input.task : null; + const semanticRunId = inputTask ? getRecordString(inputTask, "semanticRunId") : run.task.semanticRunId; + const taskId = inputTask ? getRecordString(inputTask, "taskId") : run.task.taskId; + + if (!inputTask || !semanticRunId || !taskId) { + throw new Error("SEO normalization promotion fallback не может определить semanticRunId/modelTask.taskId."); + } + + return { + modelTask: inputTask as unknown as SeoNormalizationModelTaskContract, + projectOntologyVersionId: getRecordString(inputTask, "projectOntologyVersionId"), + semanticRunId + }; +} + function getRunAiWorkspaceDispatch(run: SeoModelTaskRun): SeoAiWorkspaceDispatch | null { const fromOutput = isRecord(run.modelOutput) && isRecord(run.modelOutput.aiWorkspace) ? run.modelOutput.aiWorkspace @@ -465,11 +564,91 @@ async function promoteAiWorkspaceModelOutput( return null; } + if (run.task.taskType === "seo.context_review") { + const fallback = await getContextReviewPromotionFallback(projectId, run); + + await saveSeoContextReviewFromModelProvider( + projectId, + modelOutput.parsedJson as unknown as SeoContextReviewOutput, + run.runId, + fallback + ); + return { + nextAction: "Context Review обновлён из AI Workspace output; normalization/anchor review должны читать model-backed semantic basis.", + persistedResult: buildSeoAiWorkspacePromotedResult({ + runType: "seo_context_review", + sourceModelTaskRunId: run.runId + }) + }; + } + + if (run.task.taskType === "seo.business_synthesis") { + const inputTask = isRecord(run.input.task) ? run.input.task as unknown as SeoBusinessSynthesisModelTaskContract : null; + + if (!inputTask) { + throw new Error("Business synthesis promotion fallback не может определить modelTask."); + } + + await saveSeoBusinessSynthesisFromModelProvider( + projectId, + modelOutput.parsedJson as unknown as SeoBusinessSynthesisContract, + run.runId, + buildSeoBusinessSynthesisPromotionFallback(inputTask) + ); + return { + nextAction: "Business synthesis обновлен из AI Workspace output; normalization должна читать product-frame перед semantic basis.", + persistedResult: buildSeoAiWorkspacePromotedResult({ + runType: "seo_business_synthesis", + sourceModelTaskRunId: run.runId + }) + }; + } + + if (run.task.taskType === "seo.commercial_demand") { + const inputTask = isRecord(run.input.task) ? run.input.task as unknown as SeoCommercialDemandModelTaskContract : null; + + if (!inputTask) { + throw new Error("Commercial demand promotion fallback не может определить modelTask."); + } + + await saveSeoCommercialDemandFromModelProvider( + projectId, + modelOutput.parsedJson as unknown as SeoCommercialDemandContract, + run.runId, + buildSeoCommercialDemandPromotionFallback(inputTask) + ); + return { + nextAction: "Commercial demand обновлен из AI Workspace output; normalization должна читать buyer-intent framing перед semantic basis.", + persistedResult: buildSeoAiWorkspacePromotedResult({ + runType: "seo_commercial_demand", + sourceModelTaskRunId: run.runId + }) + }; + } + + if (run.task.taskType === "seo.market_frontier_discovery") { + await saveMarketFrontierDiscoveryFromModelProvider( + projectId, + modelOutput.parsedJson as unknown as MarketFrontierDiscoveryContract, + run.runId + ); + return { + nextAction: "Market frontier discovery обновлён из AI Workspace output; market-wide Wordstat должен читать новую память, fingerprint и probe plan.", + persistedResult: buildSeoAiWorkspacePromotedResult({ + runType: "market_frontier_discovery", + sourceModelTaskRunId: run.runId + }) + }; + } + if (run.task.taskType === "seo.normalization") { + const fallback = getNormalizationPromotionFallback(run); + await saveSeoNormalizationFromModelProvider( projectId, modelOutput.parsedJson as unknown as SeoNormalizationContract, - run.runId + run.runId, + fallback ); return { nextAction: "SEO normalization обновлена из AI Workspace output; можно пересобирать anchor review/Wordstat queue.", @@ -538,6 +717,14 @@ async function refreshRunningSeoModelTaskRun(row: SeoModelTaskRunRow) { return run; } + if (await hasNewerDoneSeoModelTaskRun(run)) { + return updateSeoModelTaskRun(run.runId, { + completed: true, + errorMessage: "Cancelled stale workspace task: a newer successful run for the same task type already exists.", + status: "cancelled" + }); + } + const dispatch = getRunAiWorkspaceDispatch(run); if (!dispatch) { @@ -615,6 +802,30 @@ async function refreshRunningSeoModelTaskRun(row: SeoModelTaskRunRow) { }); } +async function hasNewerDoneSeoModelTaskRun(run: SeoModelTaskRun) { + if (!run.task.taskType) { + return false; + } + + const result = await pool.query<{ exists: boolean }>( + ` + select exists( + select 1 + from runs + where project_id = $1 + and run_type = 'seo_model_task' + and status = 'done' + and created_at > $2::timestamptz + and output->'task'->>'taskType' = $3 + limit 1 + ) as exists + `, + [run.projectId, run.createdAt, run.task.taskType] + ); + + return result.rows[0]?.exists === true; +} + async function refreshRunningSeoModelTaskRuns(projectId: string) { const result = await pool.query( ` @@ -708,7 +919,8 @@ export async function runSeoModelTask(projectId: string, input: SeoModelTaskRunI const dispatchedRun = await updateSeoModelTaskRun(queuedRun.runId, { input: { ...baseInput, - aiWorkspace: dispatch + aiWorkspace: dispatch, + task }, output: dispatchedOutput, status: "running" diff --git a/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts b/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts index e026c25..3a3869e 100644 --- a/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts +++ b/seo_mode/seo_mode/server/src/normalization/seoNormalization.ts @@ -1,4 +1,12 @@ import { getLatestSemanticAnalysis, type SemanticAnalysisRun } from "../analysis/semanticAnalysis.js"; +import { + getSeoBusinessSynthesisContract, + type SeoBusinessSynthesisContract +} from "../business/seoBusinessSynthesis.js"; +import { + getSeoCommercialDemandContract, + type SeoCommercialDemandContract +} from "../commercial/seoCommercialDemand.js"; import { getLatestAlignedSeoContextReview, type SeoContextReviewRun @@ -31,6 +39,10 @@ type SeoNormalizationRunRow = { created_at: Date; }; +const SITE_TEXT_EVIDENCE_MAX_CHARS = 9000; +const SITE_TEXT_EVIDENCE_MAX_CHUNKS = 8; +const SITE_TEXT_EVIDENCE_MAX_LINES = 32; + export type SeoNormalizationModelTaskContract = { taskType: "seo.normalization"; schemaVersion: "seo-normalization-task.v1"; @@ -46,6 +58,23 @@ export type SeoNormalizationModelTaskContract = { topTerms: string[]; ctaPhrases: string[]; }; + siteTextEvidence: { + schemaVersion: "seo-site-text-evidence.v1"; + extraction: SemanticAnalysisRun["textCorpus"]["extraction"]; + documents: SemanticAnalysisRun["textCorpus"]["documents"]; + chunks: SemanticAnalysisRun["textCorpus"]["chunks"]; + highSignalLines: Array<{ + documentId: string; + id: string; + scope: SemanticAnalysisRun["textCorpus"]["chunks"][number]["scope"]; + selected: boolean; + sourcePath: string; + text: string; + title: string; + urlPath: string | null; + usedForCoverage: boolean; + }>; + }; contextReview: { status: "available" | "missing"; runId: string | null; @@ -62,6 +91,71 @@ export type SeoNormalizationModelTaskContract = { reviewRequired: boolean; }>; }; + businessSynthesis: { + status: "available" | "missing"; + summary: string | null; + productFrame: { + coreOffer: string; + productCategory: string; + centralThesis: string; + confidence: "low" | "medium" | "high"; + } | null; + corePillars: Array<{ + title: string; + role: string; + grounding: string; + whyItMatters: string; + }>; + applicationVectors: Array<{ + title: string; + relationshipToCore: string; + grounding: string; + priority: "high" | "medium" | "low"; + queryDirection: string; + }>; + demandFamilies: Array<{ + title: string; + role: string; + grounding: string; + priority: "high" | "medium" | "low"; + phrases: string[]; + rationale: string; + }>; + guardrails: string[]; + }; + commercialDemand: { + status: "available" | "missing"; + summary: string | null; + commercialCore: { + coreBuyerProblem: string; + coreCommercialValue: string; + coreDifferentiator: string; + buyingTrigger: string; + decisionContext: string; + confidence: "low" | "medium" | "high"; + } | null; + buyerIntentFamilies: Array<{ + title: string; + buyerIntent: string; + priority: "high" | "medium" | "low"; + grounding: string; + commercialAngle: string; + sourceMeaning: string; + coreQualifier: string; + queryPatterns: string[]; + excludedInformationalPatterns: string[]; + }>; + applicationCommercialAngles: Array<{ + sourceVector: string; + relationshipToCore: string; + priority: "high" | "medium" | "low"; + grounding: string; + commercialReframe: string; + coreQualifier: string; + queryPatterns: string[]; + }>; + guardrails: string[]; + }; scopeSummary: { indexablePages: number; selectedIndexablePages: number; @@ -218,11 +312,16 @@ export type SeoNormalizationContract = { nextActions: string[]; }; +export type SeoNormalizationModelProviderFallback = { + semanticRunId: string; + projectOntologyVersionId: string | null; + modelTask: SeoNormalizationModelTaskContract; +}; + const GENERIC_SINGLE_WORDS = new Set([ "ai", "ии", "искусственный интеллект", - "engine", "bpm", "workflow", "автоматизация", @@ -233,10 +332,73 @@ const GENERIC_SINGLE_WORDS = new Set([ "ассистент", "агент", "данные", - "задачи" + "задачи", + "доступ", + "доступа", + "роль", + "роли", + "тариф", + "тарифы" ]); const LOW_VALUE_PHRASE_PARTS = ["ла-ла", "test", "demo text", "lorem", "undefined", "null", "интент"]; +const TECHNICAL_ONLY_PHRASE_PARTS = [ + "audit trail", + "cdn", + "cloudflare", + "device posture", + "kubernetes", + "least privilege", + "mfa", + "object storage", + "postgres", + "rbac", + "redis", + "secrets vault", + "sso", + "terraform", + "waf", + "zero trust" +]; +const BUSINESS_CONTEXT_PHRASE_PARTS = [ + "агент", + "автоматизац", + "бизнес", + "данн", + "доступ", + "задач", + "инженер", + "интеграц", + "контур", + "корпоратив", + "модел", + "модул", + "платформ", + "предприят", + "приложен", + "процесс", + "рабоч", + "роль", + "согласован", + "управлен", + "workflow" +]; +const BROAD_EXPANSION_STOP_TERMS = new Set([ + "bottominfo", + "content", + "form", + "html", + "text", + "доступ", + "доступа", + "роль", + "роли", + "статус", + "статусы", + "тариф", + "тарифы", + "форма" +]); function normalizePhrase(value: string) { const cleanValue = value @@ -311,50 +473,547 @@ function mapSeoNormalizationRun(row: SeoNormalizationRunRow): SeoNormalizationCo }; } +function asRecord(value: unknown): Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)) ? (value as Record) : {}; +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function asString(value: unknown) { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function asStringArray(value: unknown): string[] { + if (typeof value === "string" && value.trim().length > 0) { + return [value.trim()]; + } + + return asArray(value) + .map(asString) + .filter((item): item is string => Boolean(item)); +} + +function asBoolean(value: unknown, fallback = false) { + return typeof value === "boolean" ? value : fallback; +} + +function normalizeConfidenceValue(value: unknown, fallback = 0.5) { + const numericValue = typeof value === "number" ? value : Number(value); + + if (!Number.isFinite(numericValue)) { + return fallback; + } + + return Math.max(0, Math.min(Number(numericValue.toFixed(2)), 1)); +} + +function normalizePriorityValue(value: unknown): "high" | "medium" | "low" { + return value === "high" || value === "medium" || value === "low" ? value : "medium"; +} + +function normalizeProviderReviewStatus(value: unknown): SeoReviewStatus { + const normalizedValue = normalizePhrase(asString(value) ?? ""); + + if (normalizedValue === "auto_approved" || normalizedValue === "approved") { + return "auto_approved"; + } + + if (normalizedValue === "rejected" || normalizedValue === "reject") { + return "rejected"; + } + + return "needs_human"; +} + +function normalizeProviderIntentType(value: unknown, phrase: string): SeoIntentType { + const normalizedValue = normalizePhrase(`${asString(value) ?? ""} ${phrase}`); + + if (/commercial|transactional|bottom|conversion|стоим|цен|тариф|демо|пилот|заяв/i.test(normalizedValue)) { + return "commercial"; + } + + if (/security|enterprise_security|безопас|доступ|роль|rbac|private|on-prem|deploy|контур/i.test(normalizedValue)) { + return "deployment_security"; + } + + if (/enterprise|корпоратив|b2b/i.test(normalizedValue)) { + return "enterprise"; + } + + if (/integration|external|api|telegram|1с|1c|crm|erp|webhook/i.test(normalizedValue)) { + return "integration"; + } + + if (/problem|business_value|операц|процесс|workflow|регламент|задач/i.test(normalizedValue)) { + return "problem"; + } + + if (/informational|faq|гайд|инструкц|как/i.test(normalizedValue)) { + return "informational"; + } + + if (/product|brand|capability|vertical|implementation|technical|модул/i.test(normalizedValue)) { + return "product"; + } + + return inferIntentType(phrase); +} + +function normalizeProviderMarketRole(value: unknown, intentType: SeoIntentType, phrase: string): SeoMarketRole { + const normalizedValue = normalizePhrase(asString(value) ?? ""); + + if (isLowValuePhrase(phrase) || /exclude|reject|noise|off_business/.test(normalizedValue)) { + return "exclude"; + } + + if (/commercial|bottom|price|tariff/.test(normalizedValue) || intentType === "commercial") { + return "commercial"; + } + + if (/integration|external_system/.test(normalizedValue) || intentType === "integration") { + return "integration"; + } + + if (/problem|business_value/.test(normalizedValue) || intentType === "problem") { + return "problem"; + } + + if (/support|broad|modifier|weak/.test(normalizedValue) || intentType === "informational" || intentType === "support") { + return "support"; + } + + return getMarketRole(phrase, intentType); +} + +function isSemanticBasisPhrase(value: string, brandName?: string | null) { + const words = normalizePhrase(value).split(/\s+/).filter(Boolean); + const brandWords = new Set(normalizePhrase(brandName ?? "").split(/\s+/).filter(Boolean)); + + if (words.length < 2 || words.length > 12) { + return false; + } + + if ( + !/[a-zа-яё0-9]/i.test(value) || + /\{\{[^}]+}}/.test(value) || + /https?:|\/\/|\.html\b|href\b|logo\b|логотип\b|beforecviz\b|staticelements\b|txt-файл|txt-файла|иконка\b/i.test(value) || + /[0-9]{6,}/.test(value) + ) { + return false; + } + + const meaningfulWords = words.filter((word) => !GENERIC_SINGLE_WORDS.has(word) && !brandWords.has(word) && word.length > 1); + + return meaningfulWords.length >= 2; +} + +function getFallbackClusters(fallback?: SeoNormalizationModelProviderFallback) { + return new Map((fallback?.modelTask.input.semanticClusters ?? []).map((cluster) => [cluster.id, cluster])); +} + +function getFallbackClusterRefs( + fallback: SeoNormalizationModelProviderFallback | undefined, + clusterIds: string[] +): SeoNormalizationContract["intentGroups"][number]["sourceRefs"] { + if (!fallback || clusterIds.length === 0) { + return []; + } + + return fallback.modelTask.input.evidenceRefs + .filter((ref) => clusterIds.some((clusterId) => ref.id.startsWith(`${clusterId}:`))) + .slice(0, 4) + .map((ref) => ({ + scope: ref.scope, + sourcePath: ref.sourcePath, + title: ref.title, + urlPath: ref.urlPath + })); +} + +function normalizeProviderSeedReview( + rawReview: unknown, + seed: Omit +): SeoNormalizationSeed["review"] { + const reviewRecord = asRecord(rawReview); + const status = normalizeProviderReviewStatus(reviewRecord.status ?? rawReview); + const sendToMarket = + status === "auto_approved" && + !seed.needsHumanReview && + seed.marketRole !== "exclude" && + seed.confidence >= 0.65; + + return { + blockers: + asStringArray(reviewRecord.blockers).length > 0 + ? asStringArray(reviewRecord.blockers) + : sendToMarket + ? [] + : ["needs_seed_review"], + reason: + asString(reviewRecord.reason) ?? + (sendToMarket + ? "Model provider предложил auto approved seed; backend оставил market routing по confidence policy." + : "Ожидает human approval: provider proposal не является разрешением на внешний сбор."), + sendToSerp: + sendToMarket && + ( + seed.intentType === "commercial" || + seed.intentType === "deployment_security" || + seed.intentType === "enterprise" || + seed.intentType === "integration" || + seed.intentType === "product" + ), + sendToWordstat: sendToMarket, + status + }; +} + +function normalizeProviderSeed( + rawSeed: unknown, + index: number, + fallback?: SeoNormalizationModelProviderFallback +): SeoNormalizationSeed | null { + const record = asRecord(rawSeed); + const phrase = asString(record.phrase) ?? asString(record.query) ?? asString(record.title); + + if (!phrase) { + return null; + } + + if (!isSemanticBasisPhrase(phrase, fallback?.modelTask.input.siteSummary.brandName)) { + return null; + } + + const clustersById = getFallbackClusters(fallback); + const clusterId = asString(record.clusterId) ?? asString(record.cluster) ?? "core_offer"; + const fallbackCluster = clustersById.get(clusterId); + const clusterTitle = asString(record.clusterTitle) ?? fallbackCluster?.title ?? clusterId; + const intentType = normalizeProviderIntentType(record.intentType, `${phrase} ${clusterTitle}`); + const marketRole = normalizeProviderMarketRole(record.marketRole, intentType, phrase); + const confidence = normalizeConfidenceValue(record.confidence, 0.5); + const reviewStatus = normalizeProviderReviewStatus(asRecord(record.review).status ?? record.review); + const needsHumanReview = asBoolean( + record.needsHumanReview, + reviewStatus !== "auto_approved" || confidence < 0.65 || marketRole === "exclude" + ); + const priority = normalizePriorityValue(record.priority ?? fallbackCluster?.priority); + const source = asString(record.source) === "ontology" ? "ontology" : "detected"; + const seedWithoutReview: Omit = { + clusterId, + clusterTitle, + confidence, + id: asString(record.id) ?? `${asString(record.intentGroupId) ?? intentType}:${clusterId}:${index}:${toId(phrase)}`, + intentGroupId: asString(record.intentGroupId) ?? `${intentType}:${clusterId}`, + intentType, + marketRole, + needsHumanReview, + normalizedFrom: asStringArray(record.normalizedFrom).length > 0 ? asStringArray(record.normalizedFrom) : [phrase], + phrase: normalizePhrase(phrase), + priority, + reason: asString(record.reason) ?? "Seed normalized from AI Workspace provider proposal.", + source + }; + + if (isTechnicalOnlySeedPhrase(seedWithoutReview.phrase)) { + return { + ...seedWithoutReview, + marketRole: "exclude", + needsHumanReview: false, + review: { + blockers: ["technical_only_phrase"], + reason: "Фраза выглядит как технический implementation/security label без самостоятельного бизнес-интента.", + sendToSerp: false, + sendToWordstat: false, + status: "rejected" + } + }; + } + + return { + ...seedWithoutReview, + review: normalizeProviderSeedReview(record.review, seedWithoutReview) + }; +} + +function getProviderRawSeeds(rawOutput: Record, fallback?: SeoNormalizationModelProviderFallback) { + const rawSeedStrategy = rawOutput.seedStrategy; + + if (Array.isArray(rawSeedStrategy)) { + return rawSeedStrategy; + } + + const seedStrategyRecord = asRecord(rawSeedStrategy); + const providerSeeds = [ + ...asArray(seedStrategyRecord.seeds), + ...asArray(seedStrategyRecord.approvedSeeds), + ...asArray(seedStrategyRecord.needsHumanReviewSeeds) + ]; + + return providerSeeds.length > 0 ? providerSeeds : fallback?.modelTask.input.sourceSeeds ?? []; +} + +function getProviderIntentGroupSeeds(rawOutput: Record) { + return asArray(rawOutput.intentGroups).flatMap((item, groupIndex) => { + const record = asRecord(item); + const groupId = asString(record.id) ?? `intent.${groupIndex + 1}`; + const sourceClusterIds = asStringArray(record.sourceClusterIds); + const clusterId = sourceClusterIds[0] ?? groupId; + const clusterTitle = asString(record.title) ?? groupId; + const intentType = asString(record.intentType); + const confidence = normalizeConfidenceValue(record.confidence, 0.55); + + return asStringArray(record.wordstatQueries).map((phrase, phraseIndex) => ({ + clusterId, + clusterTitle, + confidence, + id: `${groupId}:query:${phraseIndex + 1}:${toId(phrase)}`, + intentGroupId: groupId, + intentType, + marketRole: "market_probe", + normalizedFrom: [phrase], + phrase, + priority: normalizePriorityValue(record.priority), + reason: asString(record.marketHypothesis) ?? "Model provider normalized this phrase from an intent group.", + review: { + blockers: ["needs_seed_review"], + reason: "Ожидает human approval: intent group query не является разрешением на внешний сбор.", + sendToSerp: false, + sendToWordstat: false, + status: "needs_human" + }, + source: "detected" + })); + }); +} + +function normalizeProviderRejectedSeeds( + rawOutput: Record, + fallback?: SeoNormalizationModelProviderFallback +): SeoNormalizationContract["rejectedSeeds"] { + const clustersById = getFallbackClusters(fallback); + + return asArray(rawOutput.rejectedSeeds) + .map((item) => { + const record = asRecord(item); + const phrase = asString(record.phrase) ?? asString(record.query) ?? asString(record.title); + + if (!phrase) { + return null; + } + + const clusterId = asString(record.clusterId) ?? "core_offer"; + + return { + clusterId, + clusterTitle: asString(record.clusterTitle) ?? clustersById.get(clusterId)?.title ?? clusterId, + phrase: normalizePhrase(phrase), + reason: asString(record.reason) ?? "Provider marked seed for review/rejection.", + suggestedReplacement: asString(record.suggestedReplacement) + }; + }) + .filter((item): item is SeoNormalizationContract["rejectedSeeds"][number] => Boolean(item)); +} + +function normalizeProviderExcludedPhrases(value: unknown): SeoNormalizationContract["intentGroups"][number]["excludedPhrases"] { + return asArray(value) + .map((item) => { + const record = asRecord(item); + const phrase = typeof item === "string" ? item : asString(record.phrase) ?? asString(record.query); + + if (!phrase) { + return null; + } + + return { + phrase: normalizePhrase(phrase), + reason: asString(record.reason) ?? "Excluded by model provider." + }; + }) + .filter((item): item is SeoNormalizationContract["intentGroups"][number]["excludedPhrases"][number] => Boolean(item)); +} + +function normalizeProviderIntentGroups( + rawOutput: Record, + seedStrategy: SeoNormalizationSeed[], + fallback?: SeoNormalizationModelProviderFallback +): SeoNormalizationContract["intentGroups"] { + const rawIntentGroups = asArray(rawOutput.intentGroups); + + if (rawIntentGroups.length === 0) { + const groups = new Map(); + + for (const seed of seedStrategy) { + groups.set(seed.intentGroupId, [...(groups.get(seed.intentGroupId) ?? []), seed]); + } + + return Array.from(groups.entries()).map(([groupId, seeds]) => ({ + confidence: Math.max(...seeds.map((seed) => seed.confidence)), + corePhrases: seeds.map((seed) => seed.phrase).slice(0, 6), + excludedPhrases: [], + id: groupId, + intentType: seeds[0]?.intentType ?? "support", + marketHypothesis: "Provider proposal needs human review before market routing.", + priority: seeds[0]?.priority ?? "medium", + risks: ["Human review required before Wordstat/SERP routing."], + sourceClusterIds: Array.from(new Set(seeds.map((seed) => seed.clusterId))), + sourceRefs: getFallbackClusterRefs(fallback, Array.from(new Set(seeds.map((seed) => seed.clusterId)))), + title: seeds[0]?.clusterTitle ?? groupId, + wordstatQueries: [] + })); + } + + return rawIntentGroups.map((item, index) => { + const record = asRecord(item); + const id = asString(record.id) ?? `intent.${index + 1}`; + const groupSeeds = seedStrategy.filter((seed) => seed.intentGroupId === id); + const sourceClusterIds = + asStringArray(record.sourceClusterIds).length > 0 + ? asStringArray(record.sourceClusterIds) + : Array.from(new Set(groupSeeds.map((seed) => seed.clusterId))); + const title = asString(record.title) ?? groupSeeds[0]?.clusterTitle ?? `Intent group ${index + 1}`; + const intentType = normalizeProviderIntentType(record.intentType, `${title} ${groupSeeds.map((seed) => seed.phrase).join(" ")}`); + const wordstatQueries = asStringArray(record.wordstatQueries).map(normalizePhrase).filter(Boolean); + + return { + confidence: normalizeConfidenceValue(record.confidence, groupSeeds[0]?.confidence ?? 0.5), + corePhrases: + asStringArray(record.corePhrases).length > 0 + ? asStringArray(record.corePhrases).map(normalizePhrase).filter(Boolean).slice(0, 10) + : groupSeeds.map((seed) => seed.phrase).slice(0, 6), + excludedPhrases: normalizeProviderExcludedPhrases(record.excludedPhrases), + id, + intentType, + marketHypothesis: asString(record.marketHypothesis) ?? "Provider proposal needs human review before market routing.", + priority: normalizePriorityValue(record.priority ?? groupSeeds[0]?.priority), + risks: asStringArray(record.risks), + sourceClusterIds, + sourceRefs: getFallbackClusterRefs(fallback, sourceClusterIds), + title, + wordstatQueries + }; + }); +} + +function normalizeSeoNormalizationShape( + projectId: string, + output: SeoNormalizationContract, + fallback?: SeoNormalizationModelProviderFallback +): SeoNormalizationContract { + const rawOutput = asRecord(output); + const rawProvider = asRecord(rawOutput.provider); + const fallbackTask = fallback?.modelTask ?? (Object.keys(asRecord(rawOutput.modelTask)).length > 0 + ? (asRecord(rawOutput.modelTask) as unknown as SeoNormalizationModelTaskContract) + : null); + const providerIntentGroupSeeds = getProviderIntentGroupSeeds(rawOutput); + const rawSeeds = providerIntentGroupSeeds.length > 0 ? providerIntentGroupSeeds : getProviderRawSeeds(rawOutput, fallback); + const normalizedProviderSeeds = rawSeeds + .map((seed, index) => normalizeProviderSeed(seed, index, fallback)) + .filter((seed): seed is SeoNormalizationSeed => Boolean(seed)); + const technicalRejectedSeeds = normalizedProviderSeeds.filter((seed) => seed.review.blockers.includes("technical_only_phrase")); + const seedStrategy = normalizedProviderSeeds.filter((seed) => !seed.review.blockers.includes("technical_only_phrase")); + const rejectedSeeds = [ + ...normalizeProviderRejectedSeeds(rawOutput, fallback), + ...technicalRejectedSeeds.map((seed) => ({ + clusterId: seed.clusterId, + clusterTitle: seed.clusterTitle, + phrase: seed.phrase, + reason: seed.review.reason, + suggestedReplacement: null + })) + ]; + const intentGroups = normalizeProviderIntentGroups(rawOutput, seedStrategy, fallback); + + return { + ...output, + analystReview: output.analystReview ?? getDefaultAnalystReview(), + generatedAt: asString(rawOutput.generatedAt) ?? new Date().toISOString(), + intentGroups, + modelTask: fallbackTask, + nextActions: + asStringArray(rawOutput.nextActions).length > 0 + ? asStringArray(rawOutput.nextActions) + : [ + "Показать предложение semantic basis пользователю.", + "Не отправлять Wordstat/SERP до ручной фиксации якорей." + ], + projectId: asString(rawOutput.projectId) ?? projectId, + projectOntologyVersionId: + asString(rawOutput.projectOntologyVersionId) ?? fallback?.projectOntologyVersionId ?? fallbackTask?.projectOntologyVersionId ?? null, + provider: { + message: asString(rawProvider.message) ?? "AI Workspace provider output normalized by backend.", + mode: "codex_workspace", + modelRequired: true + }, + readiness: { + autoApprovedCount: 0, + intentGroupCount: intentGroups.length, + needsHumanReviewCount: seedStrategy.length, + normalizedSeedCount: seedStrategy.length, + rejectedSeedCount: rejectedSeeds.length, + serpApprovedCount: 0, + sourceSeedCount: fallbackTask?.input.sourceSeeds.length ?? seedStrategy.length, + wordstatApprovedCount: 0, + wordstatQueryCount: 0 + }, + rejectedSeeds, + schemaVersion: "seo-normalization.v1", + seedStrategy, + semanticRunId: asString(rawOutput.semanticRunId) ?? fallback?.semanticRunId ?? fallbackTask?.semanticRunId ?? null, + sourceTaskId: asString(rawOutput.sourceTaskId) ?? fallbackTask?.taskId ?? null, + state: seedStrategy.length > 0 ? "ready" : "not_ready", + summary: asString(rawOutput.summary) ?? "AI Workspace normalization output сохранен после backend normalization." + }; +} + function normalizeSeoNormalizationOutput( projectId: string, output: SeoNormalizationContract, providerMode: SeoNormalizationProviderMode, - providerMessage: string + providerMessage: string, + fallback?: SeoNormalizationModelProviderFallback ): SeoNormalizationContract { - if (output.schemaVersion !== "seo-normalization.v1") { + const normalizedShape = normalizeSeoNormalizationShape(projectId, output, fallback); + + if (normalizedShape.schemaVersion !== "seo-normalization.v1") { throw new Error("SEO normalization output schemaVersion должен быть seo-normalization.v1."); } - if (output.projectId !== projectId) { + if (normalizedShape.projectId !== projectId) { throw new Error("SEO normalization output projectId не совпадает с проектом."); } - if (!output.semanticRunId || !output.modelTask?.taskId) { + if (!normalizedShape.semanticRunId || !normalizedShape.modelTask?.taskId) { throw new Error("SEO normalization output должен иметь semanticRunId и modelTask.taskId."); } - if (!Array.isArray(output.intentGroups) || !Array.isArray(output.seedStrategy) || !Array.isArray(output.rejectedSeeds)) { + if (!Array.isArray(normalizedShape.intentGroups) || !Array.isArray(normalizedShape.seedStrategy) || !Array.isArray(normalizedShape.rejectedSeeds)) { throw new Error("SEO normalization output должен иметь intentGroups, seedStrategy и rejectedSeeds."); } - const seedStrategy = output.seedStrategy.map(normalizeModelSeedReview); + const seedStrategy = normalizedShape.seedStrategy.map(normalizeModelSeedReview); const wordstatApprovedCount = seedStrategy.filter((seed) => seed.review.sendToWordstat).length; const serpApprovedCount = seedStrategy.filter((seed) => seed.review.sendToSerp).length; const needsHumanReviewCount = seedStrategy.filter((seed) => seed.review.status === "needs_human").length; const autoApprovedCount = seedStrategy.filter((seed) => seed.review.status === "auto_approved").length; const rejectedSeedCount = - output.rejectedSeeds.length + seedStrategy.filter((seed) => seed.review.status === "rejected").length; + normalizedShape.rejectedSeeds.length + seedStrategy.filter((seed) => seed.review.status === "rejected").length; return { - ...output, + ...normalizedShape, generatedAt: new Date().toISOString(), provider: { message: providerMessage, mode: providerMode, modelRequired: true }, - sourceTaskId: output.modelTask.taskId, - analystReview: output.analystReview ?? getDefaultAnalystReview(), + sourceTaskId: normalizedShape.modelTask.taskId, + analystReview: normalizedShape.analystReview ?? getDefaultAnalystReview(), readiness: { - ...output.readiness, + ...normalizedShape.readiness, autoApprovedCount, - intentGroupCount: output.intentGroups.length, + intentGroupCount: normalizedShape.intentGroups.length, needsHumanReviewCount, normalizedSeedCount: seedStrategy.length, rejectedSeedCount, @@ -410,6 +1069,19 @@ function isBroadSinglePhrase(value: string) { return wordCount(phrase) <= 1 && GENERIC_SINGLE_WORDS.has(phrase); } +function isTechnicalOnlySeedPhrase(value: string) { + const phrase = normalizePhrase(value); + const technicalHitCount = TECHNICAL_ONLY_PHRASE_PARTS.filter((part) => phrase.includes(part)).length; + + if (technicalHitCount === 0) { + return false; + } + + const hasBusinessContext = BUSINESS_CONTEXT_PHRASE_PARTS.some((part) => phrase.includes(part)); + + return !hasBusinessContext || technicalHitCount >= 3; +} + function hasAny(value: string, terms: string[]) { const phrase = normalizePhrase(value); @@ -481,7 +1153,16 @@ function expandBroadPhrase(phrase: string, clusterTitle: string, siteTopTerms: s } const contextualTerms = unique(siteTopTerms) - .filter((term) => !GENERIC_SINGLE_WORDS.has(term) && wordCount(term) <= 3 && !term.includes(cleanPhrase)) + .filter((term) => { + const normalizedTerm = normalizePhrase(term); + + return ( + !GENERIC_SINGLE_WORDS.has(normalizedTerm) && + !BROAD_EXPANSION_STOP_TERMS.has(normalizedTerm) && + wordCount(normalizedTerm) <= 3 && + !normalizedTerm.includes(cleanPhrase) + ); + }) .slice(0, 3); return contextualTerms.map((term) => `${cleanPhrase} ${term}`); @@ -568,8 +1249,6 @@ const MARKET_EXPANSION_STOP_PARTS = new Set([ "bpm", "crm", "erp", - "hub", - "ops", "api", "mcp", "3d", @@ -588,21 +1267,7 @@ const MARKET_EXPANSION_STOP_PARTS = new Set([ "система" ]); -const ALLOWED_MARKET_EXPANSION_LATIN_TOKENS = new Set(["1c", "ai", "api", "bim", "bpm", "crm", "erp", "iot", "workflow"]); - -const MARKET_EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = [ - [/digital\s+twin|цифров\w*\s+двойн/i, ["цифровой двойник", "цифровые двойники", "digital twin"]], - [/\bbim\b|(^|\s)бим(\s|$)/i, ["bim", "бим"]], - [/1c|1с/i, ["1с", "1c"]], - [/закуп/i, ["закупки", "автоматизация закупок", "управление закупками"]], - [/тендер/i, ["тендеры", "тендерный помощник", "автоматизация тендеров"]], - [/юрид/i, ["юридические процессы", "юридический ассистент", "автоматизация договоров"]], - [/беспилот/i, ["беспилотная техника", "управление беспилотниками", "платформа для беспилотников"]], - [/\biot\b|интернет\s+вещ/i, ["iot", "интернет вещей", "iot платформа"]], - [/телеметр/i, ["телеметрия", "система телеметрии", "платформа телеметрии"]], - [/строител/i, ["строительство", "автоматизация строительства", "строительный контроль"]], - [/эксплуатац/i, ["эксплуатация объектов", "управление эксплуатацией", "автоматизация эксплуатации"]] -]; +const MARKET_EXPANSION_SYNONYMS: Array<[RegExp, string[]]> = []; const MARKET_EXPANSION_QUERY_BLOCKLIST = [ /(^|\s)(базов[а-яёa-z0-9-]*|трендов[а-яёa-z0-9-]*|коммерческ[а-яёa-z0-9-]*)\s+интент/i, @@ -630,17 +1295,31 @@ function getMarketExpansionSynonyms(value: string) { function hasSpecificMarketExpansionFacet(phrase: string) { const normalizedPhrase = normalizePhrase(phrase); + const aiTerms = new Set(["ai", "ии", "искусственный", "искусственного", "искусственным", "интеллект"]); + const meaningfulTerms = normalizedPhrase + .split(/\s+/) + .filter((term) => term.length > 2) + .filter((term) => !aiTerms.has(term)) + .filter((term) => !GENERIC_SINGLE_WORDS.has(term)) + .filter((term) => !MARKET_EXPANSION_STOP_PARTS.has(term)); - return /бизнес|процесс|разработ|приложен|агент|ассистент|интеграц|1с|crm|erp|bpm|workflow|цифров|двойн|bim|бим|инженер|данн|тендер|закуп|юрид|договор|беспилот|iot|телеметр|строител|эксплуатац|проект|офис|облак|оркестр|инфраструктур|безопасн/i.test( - normalizedPhrase - ); + return meaningfulTerms.length > 0; } function hasUnsafeMarketExpansionLatinToken(phrase: string) { return phrase .split(/\s+/) .flatMap((token) => token.match(/[a-z0-9-]+/gi) ?? []) - .some((token) => !ALLOWED_MARKET_EXPANSION_LATIN_TOKENS.has(token.toLowerCase())); + .some((token) => { + const normalizedToken = token.toLowerCase(); + + return ( + normalizedToken.length <= 1 || + normalizedToken.length > 14 || + /^(html|css|js|json|src|href|webp|png|jpe?g|svg|txt)$/i.test(normalizedToken) || + (normalizedToken.length <= 2 && !/\d/.test(normalizedToken)) + ); + }); } function buildPhraseExpansionVariants(phrase: string, clusterText: string) { @@ -690,7 +1369,7 @@ function isSafeWordstatExpansionPhrase(phrase: string) { hasSpecificMarketExpansionFacet(normalizedPhrase)) && !/(^|\s)(система|платформа)\s+(система|платформа)(\s|$)/i.test(normalizedPhrase) && !/искусственн[а-яёa-z0-9-]*\s+интеллект.*искусственн[а-яёa-z0-9-]*\s+интеллект/i.test(normalizedPhrase) && - !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]{1,2}$/i.test(token) && !/^(ai|bim)$/i.test(token)) + !normalizedPhrase.split(/\s+/).some((token) => /^[a-z0-9-]$/i.test(token)) ); } @@ -828,11 +1507,242 @@ function getSeedReviewDecision(seed: Omit): SeoN }; } +function countEvidenceWords(value: string) { + return normalizePhrase(value).split(/\s+/).filter(Boolean).length; +} + +function getEvidenceTextTokens(value: string) { + return normalizePhrase(value).match(/[a-zа-яё0-9-]{3,}/giu) ?? []; +} + +function getEvidenceRankingTerms(analysis: SemanticAnalysisRun) { + return unique([ + ...analysis.styleProfile.topTerms.slice(0, 40).map((term) => term.term), + ...analysis.clusters.flatMap((cluster) => [ + cluster.title, + ...cluster.matchedTerms, + ...cluster.queryExamples, + ...cluster.evidenceSnippets.slice(0, 4).flatMap((snippet) => snippet.matchedTerms) + ]) + ]) + .filter((term) => term.length >= 3 && wordCount(term) <= 5) + .slice(0, 120); +} + +function getTextEvidenceChunkScore( + chunk: SemanticAnalysisRun["textCorpus"]["chunks"][number], + rankingTerms: string[] +) { + const text = normalizePhrase(chunk.text); + const tokens = getEvidenceTextTokens(chunk.text); + const uniqueTokenCount = new Set(tokens).size; + const matchedTermScore = rankingTerms.reduce((score, term) => { + if (!text.includes(term)) { + return score; + } + + return score + (wordCount(term) > 1 ? 8 : 2); + }, 0); + const technicalPenalty = + (/\{\{[^}]+}}|<[^>]+>|\b(class|src|href|data-[\w-]+)\s*=|https?:\/\/|\.(png|jpe?g|webp|svg|gif|css|js)\b/i.test(chunk.text) + ? 30 + : 0) + (tokens.length > 0 && uniqueTokenCount / tokens.length < 0.28 ? 12 : 0); + + return ( + (chunk.selected ? 70 : 0) + + (chunk.usedForCoverage ? 90 : 0) + + (chunk.scope === "indexable_page" ? 35 : chunk.scope === "content_source" ? 25 : -25) + + Math.min(60, tokens.length / 4) + + Math.min(45, uniqueTokenCount / 2) + + matchedTermScore - + technicalPenalty + ); +} + +function getEvidenceLineCandidates(text: string) { + return text + .split(/\n+|[•●▪]/u) + .flatMap((line) => line.split(/(?<=[.!?])\s+/u)) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); +} + +function isHighSignalEvidenceLine(value: string) { + const normalizedValue = normalizePhrase(value); + const tokens = getEvidenceTextTokens(value); + + if (tokens.length < 3 || tokens.length > 16) { + return false; + } + + if ( + !/[a-zа-яё0-9]/i.test(value) || + /\{\{[^}]+}}|<[^>]+>|\b(class|src|href|data-[\w-]+)\s*=|https?:\/\/|\.(png|jpe?g|webp|svg|gif|css|js)\b/i.test(value) || + /\b(content|template|beforecviz|staticelements)\.[a-z0-9_.-]+|html\s+html|media\s+attr|boolean\s+boolean|txt-файл/iu.test(value) + ) { + return false; + } + + return tokens.some((token) => !GENERIC_SINGLE_WORDS.has(token) && !BROAD_EXPANSION_STOP_TERMS.has(token)) && normalizedValue.length <= 180; +} + +function getHighSignalEvidenceLines(chunks: SemanticAnalysisRun["textCorpus"]["chunks"]) { + const seen = new Set(); + const lines: Array< + SeoNormalizationModelTaskContract["input"]["siteTextEvidence"]["highSignalLines"][number] & { score: number } + > = []; + + chunks.forEach((chunk) => { + getEvidenceLineCandidates(chunk.text).forEach((line, index) => { + const normalizedLine = normalizePhrase(line); + + if (!isHighSignalEvidenceLine(line) || seen.has(normalizedLine)) { + return; + } + + seen.add(normalizedLine); + lines.push({ + documentId: chunk.documentId, + id: `${chunk.id}:line:${index + 1}`, + scope: chunk.scope, + selected: chunk.selected, + sourcePath: chunk.sourcePath, + text: line, + title: chunk.title, + urlPath: chunk.urlPath, + usedForCoverage: chunk.usedForCoverage, + score: + (chunk.selected ? 35 : 0) + + (chunk.usedForCoverage ? 45 : 0) + + (chunk.scope === "indexable_page" ? 20 : chunk.scope === "content_source" ? 15 : 0) + + Math.min(20, getEvidenceTextTokens(line).length) + }); + }); + }); + + return lines + .sort((left, right) => right.score - left.score || left.sourcePath.localeCompare(right.sourcePath)) + .slice(0, SITE_TEXT_EVIDENCE_MAX_LINES) + .map(({ score: _score, ...line }) => line); +} + +function buildBoundedSiteTextEvidence( + analysis: SemanticAnalysisRun +): SeoNormalizationModelTaskContract["input"]["siteTextEvidence"] { + const corpus = analysis.textCorpus; + const rankingTerms = getEvidenceRankingTerms(analysis); + const chunksById = new Map(corpus.chunks.map((chunk) => [chunk.id, chunk])); + const selectedChunkIds = new Set(); + const selectedChunks: SemanticAnalysisRun["textCorpus"]["chunks"] = []; + let selectedCharCount = 0; + + const scoredChunks = corpus.chunks + .map((chunk) => ({ + chunk, + score: getTextEvidenceChunkScore(chunk, rankingTerms) + })) + .sort((left, right) => right.score - left.score); + const scoredByDocument = new Map(); + + for (const item of scoredChunks) { + scoredByDocument.set(item.chunk.documentId, [...(scoredByDocument.get(item.chunk.documentId) ?? []), item]); + } + + const addChunk = (chunk: SemanticAnalysisRun["textCorpus"]["chunks"][number], allowLimitOverflow = false) => { + if (selectedChunkIds.has(chunk.id)) { + return; + } + + if ( + !allowLimitOverflow && + (selectedChunks.length >= SITE_TEXT_EVIDENCE_MAX_CHUNKS || + selectedCharCount + chunk.text.length > SITE_TEXT_EVIDENCE_MAX_CHARS) + ) { + return; + } + + selectedChunkIds.add(chunk.id); + selectedChunks.push(chunk); + selectedCharCount += chunk.text.length; + }; + + const priorityDocuments = [...corpus.documents] + .filter((document) => document.selected || document.usedForCoverage || document.scope === "indexable_page") + .sort((left, right) => { + const leftScore = + (left.usedForCoverage ? 3 : 0) + + (left.selected ? 2 : 0) + + (left.scope === "indexable_page" ? 1 : 0) + + Math.min(2, left.wordCount / 500); + const rightScore = + (right.usedForCoverage ? 3 : 0) + + (right.selected ? 2 : 0) + + (right.scope === "indexable_page" ? 1 : 0) + + Math.min(2, right.wordCount / 500); + + return rightScore - leftScore; + }); + + for (const document of priorityDocuments) { + const bestChunk = scoredByDocument.get(document.id)?.[0]?.chunk ?? document.chunkIds.map((id) => chunksById.get(id)).find(Boolean); + + if (bestChunk) { + addChunk(bestChunk, selectedChunks.length === 0); + } + } + + for (const item of scoredChunks) { + addChunk(item.chunk); + } + + const selectedChunksByDocument = new Map(); + + for (const chunk of selectedChunks.sort((left, right) => left.documentId.localeCompare(right.documentId) || left.chunkIndex - right.chunkIndex)) { + selectedChunksByDocument.set(chunk.documentId, [...(selectedChunksByDocument.get(chunk.documentId) ?? []), chunk]); + } + + const documents = corpus.documents + .map((document) => { + const documentChunks = selectedChunksByDocument.get(document.id) ?? []; + + if (documentChunks.length === 0) { + return null; + } + + return { + ...document, + chunkIds: documentChunks.map((chunk) => chunk.id), + textCharCount: documentChunks.reduce((sum, chunk) => sum + chunk.text.length, 0), + wordCount: documentChunks.reduce((sum, chunk) => sum + countEvidenceWords(chunk.text), 0) + }; + }) + .filter((document): document is SemanticAnalysisRun["textCorpus"]["documents"][number] => Boolean(document)); + const chunks = documents.flatMap((document) => document.chunkIds.map((id) => chunksById.get(id))).filter(Boolean) as SemanticAnalysisRun["textCorpus"]["chunks"]; + const highSignalLines = getHighSignalEvidenceLines(chunks); + + return { + schemaVersion: "seo-site-text-evidence.v1", + extraction: { + ...corpus.extraction, + chunkCount: chunks.length, + documentCount: documents.length, + sourceDocumentCount: documents.length, + totalChars: chunks.reduce((sum, chunk) => sum + chunk.text.length, 0), + totalWords: chunks.reduce((sum, chunk) => sum + countEvidenceWords(chunk.text), 0) + }, + documents, + chunks, + highSignalLines + }; +} + function buildModelTaskContract( projectId: string, analysis: SemanticAnalysisRun, projectOntologyVersionId: string | null, - contextReview: SeoContextReviewRun | null + contextReview: SeoContextReviewRun | null, + businessSynthesis: SeoBusinessSynthesisContract | null, + commercialDemand: SeoCommercialDemandContract | null ): SeoNormalizationModelTaskContract { const evidenceRefs = analysis.clusters .flatMap((cluster) => @@ -844,7 +1754,7 @@ function buildModelTaskContract( scope: ref.scope })) ) - .slice(0, 40); + .slice(0, 20); return { taskType: "seo.normalization", @@ -861,6 +1771,9 @@ function buildModelTaskContract( topTerms: analysis.styleProfile.topTerms.slice(0, 20).map((term) => term.term), ctaPhrases: analysis.styleProfile.ctaPhrases.slice(0, 10) }, + siteTextEvidence: { + ...buildBoundedSiteTextEvidence(analysis) + }, contextReview: { status: contextReview ? "available" : "missing", runId: contextReview?.runId ?? null, @@ -868,16 +1781,90 @@ function buildModelTaskContract( summary: contextReview?.summary ?? null, forbiddenClaims: contextReview?.forbiddenClaims.map((claim) => claim.claim).slice(0, 20) ?? [], normalizationHints: - contextReview?.normalizationHints.slice(0, 30).map((hint) => ({ + contextReview?.normalizationHints.slice(0, 16).map((hint) => ({ clusterId: hint.clusterId, - marketAngles: hint.marketAngles.slice(0, 8), + marketAngles: hint.marketAngles.slice(0, 5), reviewRequired: hint.reviewRequired, - sourcePhrases: hint.sourcePhrases.slice(0, 10), - stopPhrases: hint.stopPhrases.slice(0, 8), + sourcePhrases: hint.sourcePhrases.slice(0, 6), + stopPhrases: hint.stopPhrases.slice(0, 5), targetIntent: hint.targetIntent, title: hint.title })) ?? [] }, + businessSynthesis: { + status: businessSynthesis ? "available" : "missing", + summary: businessSynthesis?.summary ?? null, + productFrame: businessSynthesis + ? { + centralThesis: businessSynthesis.productFrame.centralThesis, + confidence: businessSynthesis.productFrame.confidence, + coreOffer: businessSynthesis.productFrame.coreOffer, + productCategory: businessSynthesis.productFrame.productCategory + } + : null, + corePillars: + businessSynthesis?.semanticHierarchy.corePillars.slice(0, 8).map((pillar) => ({ + grounding: pillar.grounding, + role: pillar.role, + title: pillar.title, + whyItMatters: pillar.whyItMatters + })) ?? [], + applicationVectors: + businessSynthesis?.semanticHierarchy.applicationVectors.slice(0, 10).map((vector) => ({ + grounding: vector.grounding, + priority: vector.priority, + queryDirection: vector.queryDirection, + relationshipToCore: vector.relationshipToCore, + title: vector.title + })) ?? [], + demandFamilies: + businessSynthesis?.demandFamilies.slice(0, 12).map((family) => ({ + grounding: family.grounding, + phrases: family.phrases.slice(0, 6), + priority: family.priority, + rationale: family.rationale, + role: family.role, + title: family.title + })) ?? [], + guardrails: businessSynthesis?.guardrails.slice(0, 12) ?? [] + }, + commercialDemand: { + status: commercialDemand ? "available" : "missing", + summary: commercialDemand?.summary ?? null, + commercialCore: commercialDemand + ? { + buyingTrigger: commercialDemand.commercialCore.buyingTrigger, + confidence: commercialDemand.commercialCore.confidence, + coreBuyerProblem: commercialDemand.commercialCore.coreBuyerProblem, + coreCommercialValue: commercialDemand.commercialCore.coreCommercialValue, + coreDifferentiator: commercialDemand.commercialCore.coreDifferentiator, + decisionContext: commercialDemand.commercialCore.decisionContext + } + : null, + buyerIntentFamilies: + commercialDemand?.buyerIntentFamilies.slice(0, 12).map((family) => ({ + buyerIntent: family.buyerIntent, + commercialAngle: family.commercialAngle, + coreQualifier: family.coreQualifier, + excludedInformationalPatterns: family.excludedInformationalPatterns.slice(0, 5), + grounding: family.grounding, + priority: family.priority, + queryPatterns: family.queryPatterns.slice(0, 8), + sourceMeaning: family.sourceMeaning, + title: family.title + })) ?? [], + applicationCommercialAngles: + commercialDemand?.applicationCommercialAngles.slice(0, 12).map((angle) => ({ + commercialReframe: angle.commercialReframe, + coreQualifier: angle.coreQualifier, + grounding: angle.grounding, + priority: angle.priority, + queryPatterns: angle.queryPatterns.slice(0, 6), + relationshipToCore: angle.relationshipToCore, + sourceVector: angle.sourceVector + })) ?? [], + guardrails: commercialDemand?.guardrails.slice(0, 12) ?? [] + }, scopeSummary: { adminPages: analysis.pageScope.adminPages, contentSources: analysis.pageScope.contentSources, @@ -887,29 +1874,29 @@ function buildModelTaskContract( templateBlocks: analysis.pageScope.templateBlocks }, evidenceRefs, - semanticClusters: analysis.clusters.slice(0, 24).map((cluster) => ({ + semanticClusters: analysis.clusters.slice(0, 16).map((cluster) => ({ evidenceLevel: cluster.evidence?.level ?? "none", id: cluster.id, - matchedTerms: cluster.matchedTerms.slice(0, 12), + matchedTerms: cluster.matchedTerms.slice(0, 8), priority: cluster.priority, - queryExamples: cluster.queryExamples.slice(0, 8), + queryExamples: cluster.queryExamples.slice(0, 5), status: cluster.status, targetIntent: cluster.targetIntent, title: cluster.title })), verticalOpportunities: analysis.clusters .filter((cluster) => cluster.status !== "missing" || cluster.priority === "high") - .slice(0, 24) + .slice(0, 12) .map((cluster) => ({ id: cluster.id, - matchedTerms: cluster.matchedTerms.slice(0, 16), + matchedTerms: cluster.matchedTerms.slice(0, 8), priority: cluster.priority, - queryExamples: cluster.queryExamples.slice(0, 12), - sourceTitles: cluster.sourceRefs.slice(0, 8).map((ref) => ref.title), - sourceSnippets: cluster.evidenceSnippets.slice(0, 8).map((snippet) => snippet.text), + queryExamples: cluster.queryExamples.slice(0, 5), + sourceTitles: cluster.sourceRefs.slice(0, 4).map((ref) => ref.title), + sourceSnippets: cluster.evidenceSnippets.slice(0, 3).map((snippet) => snippet.text.slice(0, 180)), status: cluster.status, targetIntent: cluster.targetIntent, - targetPages: cluster.targetPages.slice(0, 8), + targetPages: cluster.targetPages.slice(0, 4), title: cluster.title })), sourceSeeds: analysis.seedCandidates.slice(0, 80).map((seed) => ({ @@ -958,6 +1945,13 @@ function buildModelTaskContract( "Фраза выглядит как соседний рынок или business expansion без явного user approval.", "Не отбрасывать source-grounded vertical opportunity: если точной рыночной формулировки нет, вернуть needs_human seed или expansion query, а не rejected.", "Для каждой видимой продуктовой/отраслевой вертикали предложить 2-6 Wordstat-safe аналогов: коротких, рыночных, без внутренних labels.", + "Не пропускать non-technical строки из siteTextEvidence.highSignalLines: представить их как needs_human query/corePhrase или явно объяснить exclusion.", + "Business synthesis is the hierarchy source: core/product families must stay centered on productFrame and corePillars; applicationVectors must be labeled as use cases or verticals, not promoted to core.", + "Commercial demand is the buyer-intent source: do not output raw architecture theses as seedStrategy phrases unless reframed as buyer problem, business value, solution search, implementation, vendor selection, integration, pilot, replacement, or commercial evaluation.", + "seedStrategy must contain the actual semantic-basis query candidates, not only intent group corePhrases: target 24-60 needs_human seeds across core buyer-intent groups and application angles.", + "Keep seedStrategy phrases concise enough for market review, usually 3-10 words; if a buyer-intent phrase is longer, rewrite it shorter while preserving the commercial value and core differentiator.", + "Do not shorten by deleting the core differentiator from commercialDemand.commercialCore/coreQualifier. A 9-12 word qualified buyer query is better than a 4-6 word generic phrase for another market.", + "For application vectors, keep the product core differentiator in query patterns when detaching it would create another market.", "Нельзя определить, отправлять ли фразу в Wordstat/SERP без ручного review." ] }; @@ -987,7 +1981,9 @@ function buildContract( projectId: string, analysis: SemanticAnalysisRun | null, projectOntologyVersionId: string | null, - contextReview: SeoContextReviewRun | null + contextReview: SeoContextReviewRun | null, + businessSynthesis: SeoBusinessSynthesisContract | null, + commercialDemand: SeoCommercialDemandContract | null ): SeoNormalizationContract { if (!analysis) { return { @@ -1155,7 +2151,7 @@ function buildContract( const wordstatApprovedCount = normalizedSeedStrategy.filter((seed) => seed.review.sendToWordstat).length; const serpApprovedCount = normalizedSeedStrategy.filter((seed) => seed.review.sendToSerp).length; const reviewRejectedCount = normalizedSeedStrategy.filter((seed) => seed.review.status === "rejected").length; - const modelTask = buildModelTaskContract(projectId, analysis, projectOntologyVersionId, contextReview); + const modelTask = buildModelTaskContract(projectId, analysis, projectOntologyVersionId, contextReview, businessSynthesis, commercialDemand); return { schemaVersion: "seo-normalization.v1", @@ -1207,18 +2203,32 @@ export async function getSeoNormalizationContract(projectId: string): Promise { const normalizedOutput = normalizeSeoNormalizationOutput( projectId, output, "codex_workspace", - "AI Workspace Codex provider выполнил seo.normalization по contract-only task; результат сохранён как evidence после backend validation." + "AI Workspace Codex provider выполнил seo.normalization по contract-only task; результат сохранён как evidence после backend validation.", + fallback ); const result = await pool.query( ` diff --git a/seo_mode/seo_mode/server/src/ontology/projectOntology.ts b/seo_mode/seo_mode/server/src/ontology/projectOntology.ts index e2e304c..09d77ac 100644 --- a/seo_mode/seo_mode/server/src/ontology/projectOntology.ts +++ b/seo_mode/seo_mode/server/src/ontology/projectOntology.ts @@ -253,20 +253,67 @@ const INTENT_SIGNAL_GROUP_META = { const ONTOLOGY_STOP_WORDS = new Set([ "and", "app", + "assets", + "attr", + "bodyhtml", + "bottominfo", + "card", + "collection", "com", + "content", + "cta", + "cviz", + "form", "для", + "enabled", + "features", + "footer", + "heading", + "html", + "image", + "images", + "introhtml", + "json", "или", "как", "компания", + "main", + "media", + "muted", "на", "об", "от", "по", "под", "при", + "scope", + "src", + "static", + "staticelements", + "target", + "text", + "txt", + "txt-файл", + "txt-файла", + "txt-файлы", "сайт", "сервис", "страница", + "webp", + "аватар", + "блок", + "заголовок", + "иконка", + "информационный", + "карточка", + "основная", + "приглушенная", + "пункт", + "пункты", + "строка", + "текст", + "форма", + "ячейка", "это" ]); @@ -337,11 +384,12 @@ function getTextWords(value: string) { return normalizeOntologyText(value).match(/[a-zа-я0-9-]{3,}/giu) ?? []; } -function getTopTerms(value: string, limit: number) { +function getTopTerms(value: string, limit: number, extraStopTerms: string[] = []) { const counts = new Map(); + const stopTerms = new Set([...extraStopTerms.map(normalizeOntologyText), ...ONTOLOGY_STOP_WORDS]); for (const word of getTextWords(value)) { - if (ONTOLOGY_STOP_WORDS.has(word) || /^\d+$/.test(word) || /^[0-9a-f]{12,}$/i.test(word)) { + if (stopTerms.has(word) || /^\d+$/.test(word) || /^[0-9a-f]{12,}$/i.test(word)) { continue; } @@ -526,7 +574,7 @@ function buildGenericLandingBrief(brandName: string, documents: ProjectOntologyE function buildCoreOfferQueries(brandName: string, topTerms: string[]) { const evidenceTerms = topTerms.filter((term) => normalizeOntologyText(term) !== normalizeOntologyText(brandName)).slice(0, 5); - const pairedTerms = evidenceTerms.slice(0, 3).map((term) => `${term} ${brandName}`); + const pairedTerms = evidenceTerms.slice(0, 3).map((term) => `${brandName} ${term}`); const topicPair = evidenceTerms.length >= 2 ? `${evidenceTerms[0]} ${evidenceTerms[1]}` : null; return dedupeStrings([brandName, ...pairedTerms, topicPair ?? ""]).slice(0, 4); @@ -534,7 +582,8 @@ function buildCoreOfferQueries(brandName: string, topTerms: string[]) { function buildEvidenceSeedQueries(brandName: string, matchedTerms: string[], topTerms: string[]) { const brandKey = normalizeOntologyText(brandName); - const terms = dedupeStrings([...matchedTerms, ...topTerms]) + const evidenceTerms = matchedTerms.length > 0 ? matchedTerms : topTerms; + const terms = dedupeStrings(evidenceTerms) .filter((term) => { const normalizedTerm = normalizeOntologyText(term); @@ -557,7 +606,8 @@ function buildEvidenceSeedQueries(brandName: string, matchedTerms: string[], top function buildExtractedClusters(evidence: ProjectOntologyEvidence, brandName: string) { const documents = getCoverageDocuments(evidence); const allText = normalizeOntologyText(documents.map((document) => document.text).join("\n")); - const topTerms = getTopTerms(allText, 10); + const brandTerms = buildBrandTerms(brandName, evidence); + const topTerms = getTopTerms(allText, 10, brandTerms); const homeDocument = documents.find((document) => document.kind === "page") ?? documents[0] ?? null; const homeTarget = homeDocument ? getPagePath(homeDocument) : "/"; const coreKeywords = dedupeStrings([...topTerms.slice(0, 8), ...buildBrandTerms(brandName, evidence)]).slice(0, 10); @@ -598,7 +648,7 @@ function buildExtractedClusters(evidence: ProjectOntologyEvidence, brandName: st priority, targetIntent: meta.targetIntent, intentGroups: [groupId], - keywords: dedupeStrings([...matchedTerms, ...topTerms.slice(0, 4)]).slice(0, 12), + keywords: dedupeStrings([...matchedTerms, ...(matchedTerms.length >= 4 ? [] : topTerms.slice(0, 4))]).slice(0, 12), queryExamples: seedQueries.length > 0 ? seedQueries : buildCoreOfferQueries(brandName, topTerms), targetPages: targetPages.length > 0 ? targetPages : [homeTarget] }); diff --git a/seo_mode/seo_mode/server/src/projects/routes.ts b/seo_mode/seo_mode/server/src/projects/routes.ts index 246121e..4a1e9d9 100644 --- a/seo_mode/seo_mode/server/src/projects/routes.ts +++ b/seo_mode/seo_mode/server/src/projects/routes.ts @@ -47,7 +47,11 @@ import { saveSeoContextReviewManual, type SeoContextReviewOutput } from "../contextReview/seoContextReview.js"; -import { getMarketEnrichmentContract, runMarketEnrichment } from "../market/marketEnrichment.js"; +import { + getMarketEnrichmentContract, + getWordstatProbePlanPreview, + runMarketEnrichment +} from "../market/marketEnrichment.js"; import { getSeoModelContextContract } from "../modelContext/seoModelContext.js"; import { getSeoStrategyContract } from "../strategy/seoStrategy.js"; import { @@ -90,6 +94,17 @@ import { saveKeywordCleaningManual, type KeywordCleaningContract } from "../keywords/keywordCleaning.js"; +import { + getLatestKeywordAnalysisWorkflow, + startKeywordAnalysisWorkflow +} from "../keywords/keywordAnalysisWorkflow.js"; +import { + deleteKeywordContextSnapshot, + listKeywordContextSnapshots, + renameKeywordContextSnapshot, + saveKeywordContextSnapshot, + updateKeywordContextSnapshotCuration +} from "../keywords/keywordContextSnapshots.js"; import { addAnchorReviewManualAnchor, getAnchorReviewContract, @@ -235,8 +250,43 @@ const keywordMapApproveSchema = z.object({ role: z.enum(["differentiator", "primary", "secondary", "secondary_candidate", "support", "validate"]) }) ) + .optional(), + suppressedItemIds: z.array(z.string().min(1)).max(300).optional() +}); +const keywordContextSnapshotSchema = z.object({ + label: z.string().trim().max(180).optional(), + note: z.string().trim().max(1000).optional(), + source: z.enum(["manual_save", "expansion_checkpoint"]).optional(), + curation: z + .object({ + roleOverrides: z + .array( + z.object({ + itemId: z.string().min(1), + role: z.string().min(1) + }) + ) + .max(1000) + .optional(), + suppressedItemIds: z.array(z.string().min(1)).max(1000).optional() + }) .optional() }); +const keywordContextSnapshotRenameSchema = z.object({ + label: z.string().trim().min(1, "Нужно указать название snapshot-профиля.").max(180) +}); +const keywordContextSnapshotCurationSchema = z.object({ + roleOverrides: z + .array( + z.object({ + itemId: z.string().min(1), + role: z.string().min(1) + }) + ) + .max(1000) + .optional(), + suppressedItemIds: z.array(z.string().min(1)).max(1000).optional() +}); const contextReviewManualSchema = z.object({ output: z.record(z.string(), z.unknown()) }); @@ -251,10 +301,12 @@ const anchorReviewDecisionSchema = z.object({ reason: z.string().trim().max(1000).optional(), sendToSerp: z.boolean().optional(), sendToWordstat: z.boolean().optional(), - status: z.enum(["approved", "disabled", "pending"]) + status: z.enum(["approved", "deleted", "disabled", "pending"]) }); +const demandCollectionProfileSchema = z.enum(["contextual", "market_wide"]); const anchorReviewDecisionsSchema = z.object({ - decisions: z.array(anchorReviewDecisionSchema).max(200).optional() + decisions: z.array(anchorReviewDecisionSchema).max(200).optional(), + demandCollectionProfile: demandCollectionProfileSchema.optional() }); const anchorReviewManualAnchorSchema = z.object({ phrase: z.string().trim().min(1).max(180), @@ -768,6 +820,24 @@ projectsRouter.get("/:projectId/market-enrichment/latest", async (request, respo } }); +projectsRouter.get("/:projectId/market-enrichment/probe-preview", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + + response.json({ + probePreview: await getWordstatProbePlanPreview(projectId) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта."); + return; + } + + console.error(error); + sendError(response, 500, "Не удалось построить preview Wordstat probe plan."); + } +}); + projectsRouter.get("/:projectId/seo-normalization/latest", async (request, response) => { try { const projectId = projectIdSchema.parse(request.params.projectId); @@ -829,7 +899,7 @@ projectsRouter.post("/:projectId/anchor-review/decisions", async (request, respo const input = anchorReviewDecisionsSchema.parse(request.body ?? {}); response.status(201).json({ - anchorReview: await saveAnchorReviewDecisions(projectId, input.decisions ?? []) + anchorReview: await saveAnchorReviewDecisions(projectId, input.decisions, input.demandCollectionProfile) }); } catch (error) { if (error instanceof z.ZodError) { @@ -879,6 +949,42 @@ projectsRouter.get("/:projectId/keyword-cleaning/latest", async (request, respon } }); +projectsRouter.get("/:projectId/keyword-analysis-workflow/latest", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + + response.json({ + workflow: await getLatestKeywordAnalysisWorkflow(projectId) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта."); + return; + } + + console.error(error); + sendError(response, 500, "Не удалось загрузить workflow анализа ключей."); + } +}); + +projectsRouter.post("/:projectId/keyword-analysis-workflow", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + + response.status(202).json({ + workflow: await startKeywordAnalysisWorkflow(projectId) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта."); + return; + } + + console.error(error); + sendError(response, 500, error instanceof Error ? error.message : "Не удалось запустить workflow анализа ключей."); + } +}); + projectsRouter.post("/:projectId/keyword-cleaning/manual", async (request, response) => { try { const projectId = projectIdSchema.parse(request.params.projectId); @@ -980,6 +1086,101 @@ projectsRouter.get("/:projectId/keyword-map/latest", async (request, response) = } }); +projectsRouter.get("/:projectId/keyword-context-snapshots", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + + response.json({ + snapshots: await listKeywordContextSnapshots(projectId) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Некорректный id проекта."); + return; + } + + console.error(error); + sendError(response, 500, "Не удалось загрузить snapshot-профили ключей."); + } +}); + +projectsRouter.post("/:projectId/keyword-context-snapshots", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + const input = keywordContextSnapshotSchema.parse(request.body ?? {}); + + response.status(201).json({ + snapshot: await saveKeywordContextSnapshot(projectId, input) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Проверь snapshot payload."); + return; + } + + console.error(error); + sendError(response, 500, error instanceof Error ? error.message : "Не удалось сохранить snapshot ключей."); + } +}); + +projectsRouter.patch("/:projectId/keyword-context-snapshots/:snapshotId", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + const snapshotId = z.string().uuid("Некорректный id snapshot-профиля.").parse(request.params.snapshotId); + const input = keywordContextSnapshotRenameSchema.parse(request.body ?? {}); + + response.json({ + snapshot: await renameKeywordContextSnapshot(projectId, snapshotId, input.label) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Проверь snapshot payload."); + return; + } + + console.error(error); + sendError(response, 500, error instanceof Error ? error.message : "Не удалось переименовать snapshot ключей."); + } +}); + +projectsRouter.patch("/:projectId/keyword-context-snapshots/:snapshotId/curation", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + const snapshotId = z.string().uuid("Некорректный id snapshot-профиля.").parse(request.params.snapshotId); + const input = keywordContextSnapshotCurationSchema.parse(request.body ?? {}); + + response.json({ + snapshot: await updateKeywordContextSnapshotCuration(projectId, snapshotId, input) + }); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Проверь layout snapshot payload."); + return; + } + + console.error(error); + sendError(response, 500, error instanceof Error ? error.message : "Не удалось сохранить раскладку ключей."); + } +}); + +projectsRouter.delete("/:projectId/keyword-context-snapshots/:snapshotId", async (request, response) => { + try { + const projectId = projectIdSchema.parse(request.params.projectId); + const snapshotId = z.string().uuid("Некорректный id snapshot-профиля.").parse(request.params.snapshotId); + + await deleteKeywordContextSnapshot(projectId, snapshotId); + response.status(204).send(); + } catch (error) { + if (error instanceof z.ZodError) { + sendError(response, 400, error.issues[0]?.message ?? "Некорректный id snapshot-профиля."); + return; + } + + console.error(error); + sendError(response, 500, error instanceof Error ? error.message : "Не удалось удалить snapshot ключей."); + } +}); + projectsRouter.get("/:projectId/strategy/latest", async (request, response) => { try { const projectId = projectIdSchema.parse(request.params.projectId); diff --git a/seo_mode/seo_mode/server/src/settings/externalServiceSettings.ts b/seo_mode/seo_mode/server/src/settings/externalServiceSettings.ts index 6937d11..1ab6ee1 100644 --- a/seo_mode/seo_mode/server/src/settings/externalServiceSettings.ts +++ b/seo_mode/seo_mode/server/src/settings/externalServiceSettings.ts @@ -256,7 +256,7 @@ const serviceDefinitions: Record = label: "Max seeds per run", type: "number", required: false, - placeholder: "40", + placeholder: "12", help: "Сколько seed-фраз максимум отправлять за один запуск Wordstat, чтобы не сжечь внешний бюджет." }, { @@ -422,7 +422,7 @@ function getFieldDefault(serviceId: ExternalServiceId, fieldId: string) { apiBaseUrl: WORDSTAT_BASE_URL, devices: "DEVICE_ALL", numPhrases: "50", - maxSeedsPerRun: "40", + maxSeedsPerRun: "12", regionIds: "225" }; @@ -772,7 +772,7 @@ export async function getWordstatRuntimeConfig(): Promise provider: "mcp_kv", mode: "mcp_kv", mcpEndpoint: endpoint, - maxSeedsPerRun: getNumber(mapped.publicConfig.maxSeedsPerRun, 40) + maxSeedsPerRun: getNumber(mapped.publicConfig.maxSeedsPerRun, 12) }; } } @@ -793,7 +793,7 @@ export async function getWordstatRuntimeConfig(): Promise regionIds: splitCsv(mapped.publicConfig.regionIds, ["225"]), devices: splitCsv(mapped.publicConfig.devices, ["DEVICE_ALL"]), numPhrases: getNumber(mapped.publicConfig.numPhrases, 50), - maxSeedsPerRun: getNumber(mapped.publicConfig.maxSeedsPerRun, 40) + maxSeedsPerRun: getNumber(mapped.publicConfig.maxSeedsPerRun, 12) }; } diff --git a/seo_mode/seo_mode/server/src/skills/seoSkillRegistry.ts b/seo_mode/seo_mode/server/src/skills/seoSkillRegistry.ts index f0af33f..a64e51a 100644 --- a/seo_mode/seo_mode/server/src/skills/seoSkillRegistry.ts +++ b/seo_mode/seo_mode/server/src/skills/seoSkillRegistry.ts @@ -118,6 +118,46 @@ export type SeoContextReviewModelTaskContract = { wordCount: number; matchedClusterIds: string[]; }>; + siteTextEvidence: { + schemaVersion: "seo-site-text-evidence.v1"; + extraction: { + chunkCount: number; + documentCount: number; + mode: "clean_visible_text"; + sourceDocumentCount: number; + totalChars: number; + totalWords: number; + }; + documents: Array<{ + chunkIds: string[]; + id: string; + kind: "page" | "content_json"; + scope: string; + selected: boolean; + sourcePath: string; + textCharCount: number; + title: string; + urlPath: string | null; + usedForCoverage: boolean; + wordCount: number; + }>; + chunks: Array<{ + charEnd: number; + charStart: number; + chunkIndex: number; + documentId: string; + id: string; + kind: "page" | "content_json"; + scope: string; + selected: boolean; + sourcePath: string; + text: string; + title: string; + totalChunks: number; + urlPath: string | null; + usedForCoverage: boolean; + }>; + }; semanticSignals: Array<{ id: string; title: string; @@ -224,12 +264,13 @@ const SEO_SKILL_MANIFESTS: SeoSkillManifest[] = [ summary: "Первый полноценный AI entrypoint: модель объясняет смысл сайта, страницы, оффера, аудитории, gaps, ambiguity и forbidden claims по evidence.", owner: "seo_mode", - inputs: ["semantic_analysis", "project_ontology", "page_scope", "style_profile", "crawl_indexability_evidence"], - requiredEvidence: ["semantic_clusters", "page_signals", "content_sources", "source_refs", "style_profile"], + inputs: ["semantic_analysis", "project_ontology", "page_scope", "style_profile", "crawl_indexability_evidence", "site_text_evidence"], + requiredEvidence: ["clean_visible_text_chunks", "semantic_clusters", "page_signals", "content_sources", "source_refs", "style_profile"], deterministicChecks: [ { id: "analysis_ready", title: "Semantic analysis exists", source: "backend", required: true }, { id: "scope_ready", title: "Scope summary exists", source: "backend", required: true }, - { id: "evidence_refs", title: "Evidence refs are attached to semantic claims", source: "backend", required: true } + { id: "evidence_refs", title: "Evidence refs are attached to semantic claims", source: "backend", required: true }, + { id: "site_text", title: "Clean selected site text chunks are attached", source: "backend", required: true } ], modelTasks: [ { @@ -548,10 +589,37 @@ function getEvidenceRefs(analysis: SemanticAnalysisRun): SeoContextReviewModelTa .slice(0, 60); } +function getTextCorpusEvidence(analysis: SemanticAnalysisRun): SeoContextReviewModelTaskContract["input"]["siteTextEvidence"] { + if (analysis.textCorpus) { + return { + schemaVersion: "seo-site-text-evidence.v1", + extraction: analysis.textCorpus.extraction, + documents: analysis.textCorpus.documents, + chunks: analysis.textCorpus.chunks + }; + } + + return { + schemaVersion: "seo-site-text-evidence.v1", + extraction: { + chunkCount: 0, + documentCount: 0, + mode: "clean_visible_text", + sourceDocumentCount: 0, + totalChars: 0, + totalWords: 0 + }, + documents: [], + chunks: [] + }; +} + export function buildSeoContextReviewTask( projectId: string, analysis: SemanticAnalysisRun ): SeoContextReviewModelTaskContract { + const siteTextEvidence = getTextCorpusEvidence(analysis); + return { taskType: "seo.context_review", schemaVersion: "seo-context-review-task.v1", @@ -597,6 +665,7 @@ export function buildSeoContextReviewTask( wordCount: source.wordCount, matchedClusterIds: source.matchedClusters })), + siteTextEvidence, semanticSignals: analysis.clusters.slice(0, 30).map((cluster) => ({ evidenceLevel: cluster.evidence.level, id: cluster.id, @@ -657,7 +726,7 @@ export function buildSeoContextReviewTask( requireHumanReviewWhenAmbiguous: true }, stopConditions: [ - "Недостаточно evidence refs для определения реального оффера сайта.", + "Недостаточно clean text evidence или evidence refs для определения реального оффера сайта.", "Сайт выражен слишком слабо, и модель не может отделить продукт от технической оболочки.", "Запрошенное действие требует business expansion, rewrite или apply, которые запрещены на context review stage." ]