АДРЕСНЫЙ РЕЖИМ - ллм декомпоз

This commit is contained in:
2026-04-01 22:11:40 +03:00
parent 18e0f1364d
commit b2d32f869c
1540 changed files with 10464 additions and 1451 deletions
+102 -17
View File
@@ -1,6 +1,10 @@
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractAddressFilters = extractAddressFilters;
const iconv_lite_1 = __importDefault(require("iconv-lite"));
const ACCOUNT_PATTERN = /(?:сч[её]т|счет|account)[^0-9]{0,12}(\d{2}(?:[.,]\d{1,2})?)/i;
const LIMIT_PATTERN = /(?:\btop\b|\blimit\b|\bпервые\b|\bтоп\b)\s*(\d{1,3})/i;
const COUNTERPARTY_PATTERN = /(?:по\s+контрагенту|контрагент(?:у|а)?|по\s+контре|контра|по\s+компан(?:ии|ию|ия)|компан(?:ия|ии|ию)|по\s+организац(?:ии|ию|ия)|организац(?:ия|ии|ию)|по\s+поставщик(?:у|а)?|поставщик(?:у|а)?|по\s+клиент(?:у|а)?|клиент(?:у|а)?|по\s+покупател(?:ю|я)|покупател(?:ю|я)|по\s+партнер(?:у|а)?|партнер(?:у|а)?|by\s+counterparty|counterparty|by\s+company|company|by\s+supplier|supplier|by\s+vendor|vendor|by\s+customer|customer|by\s+client|client|by\s+partner|partner)\s+([^\r\n,.;:]+)/iu;
@@ -11,13 +15,62 @@ const PERIOD_RANGE_PATTERN_1 = /(?:from|с)\s+(\d{1,4}[.\/-]\d{1,2}[.\/-]\d{1,4}
const PERIOD_RANGE_PATTERN_2 = /(?:between|за\s+период\s+с)\s+(\d{1,4}[.\/-]\d{1,2}[.\/-]\d{1,4})\s+(?:and|по)\s+(\d{1,4}[.\/-]\d{1,2}[.\/-]\d{1,4})/i;
const YEAR_RANGE_PATTERN = /(?:за|for|с|from)?\s*(20\d{2})\s*(?:[-‐‑‒–—―−]|до|to|по)\s*(20\d{2})(?:\s*(?:г(?:од|ода)?\.?|year))?(?=[^\d]|$)/iu;
const YEAR_RANGE_LOOSE_PATTERN = /\b(20\d{2})\b\s*(?:[-‐‑‒–—―−]|до|to|по)\s*\b(20\d{2})\b/iu;
const YEAR_PERIOD_PATTERN = /(?:за|for)\s*(20\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year)?/iu;
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year)(?=$|[\s,.;:!?()\-])/iu;
const YEAR_PERIOD_PATTERN = /(?:за|for)\s*(20\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year|god)?/iu;
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year|god)(?=$|[\s,.;:!?()\-])/iu;
const YEAR_PERIOD_SHORT_ORDINAL_PATTERN = /(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(\d{2})\s*(?:[-\s]?(?:й|ый|ой|th))(?:\s*(?:г(?:од|ода)?\.?|year|period|период))?(?=$|[\s,.;:!?()\-])/iu;
const YEAR_PERIOD_ANY_PATTERN = /(?:^|[\s,.;:!?()\-])((?:19|20)\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*(?:19|20)\d{2})(?![.\/-]\d)(?:\s*(?:г(?:од|ода)?\.?|year))?(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NUMERIC_MONTH_YEAR_PATTERN = /(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(0?[1-9]|1[0-2])[.\/-](20\d{2})(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NUMERIC_YEAR_MONTH_PATTERN = /(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(20\d{2})[.\/-](0?[1-9]|1[0-2])(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NAME_PATTERN = /(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*([a-zа-яё]+)\s+(20\d{2})(?:\s*г(?:од|ода|\\.)?)?(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NAME_YEAR_FIRST_PATTERN = /(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(20\d{2})(?:\s*г(?:од|ода|\\.)?)?\s+([a-zа-яё]+)(?=$|[\s,.;:!?()\-])/iu;
const DOC_SIGNAL_PATTERN = "(?:док(?:и|умент|ументы|ументов|умам|ума)|docs?|documents?|doki|dokument(?:y|ov|am|a)?)";
function textMojibakeScore(value) {
const source = String(value ?? "");
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
}
function looksLikeMojibake(value) {
const source = String(value ?? "");
if (!source.trim()) {
return false;
}
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
return true;
}
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
}
function decodeUtf8FromWin1251Mojibake(value) {
if (!looksLikeMojibake(value)) {
return value;
}
try {
const bytes = iconv_lite_1.default.encode(value, "win1251");
const decoded = bytes.toString("utf8");
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
}
catch {
return value;
}
}
function decodeUtf8FromLatin1Mojibake(value) {
if (!looksLikeMojibake(value)) {
return value;
}
try {
const decoded = Buffer.from(value, "latin1").toString("utf8");
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
}
catch {
return value;
}
}
function normalizeMojibakeString(value) {
const fromWin1251 = decodeUtf8FromWin1251Mojibake(value);
return decodeUtf8FromLatin1Mojibake(fromWin1251);
}
function toIsoDate(year, month, day) {
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
return null;
@@ -187,18 +240,28 @@ function extractYearPeriod(text) {
}
}
const shortYearMatch = text.match(YEAR_PERIOD_SHORT_PATTERN);
if (!shortYearMatch) {
return {};
if (shortYearMatch) {
const shortYear = Number(shortYearMatch[1]);
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
}
}
const shortYear = Number(shortYearMatch[1]);
if (!Number.isFinite(shortYear) || shortYear < 0 || shortYear > 99) {
return {};
const shortOrdinalMatch = text.match(YEAR_PERIOD_SHORT_ORDINAL_PATTERN);
if (shortOrdinalMatch) {
const shortYear = Number(shortOrdinalMatch[1]);
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
}
}
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
return {};
}
function extractYearRangePeriod(text) {
const match = text.match(YEAR_RANGE_PATTERN) ?? text.match(YEAR_RANGE_LOOSE_PATTERN);
@@ -347,13 +410,32 @@ function isLikelyCounterpartyToken(rawToken) {
"нахуй",
"покеж",
"покажи",
"выведи"
"показать",
"выведи",
"show",
"list",
"please",
"vse",
"all",
"kakie",
"kakoi",
"est",
"pokaji",
"pokazhi",
"pokazh",
"pokezh",
"doki",
"doky",
"dokument",
"dokumenty",
"documents",
"docs"
]);
return !stopWords.has(lowered);
}
function hasDocsOrBankSignal(text) {
const lowered = String(text ?? "").toLowerCase();
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|transactions?|bank\s+ops|bank\s+operations?)/iu.test(lowered);
return new RegExp(`(?:${DOC_SIGNAL_PATTERN}|банк|выписк|платеж|платёж|оплат|transactions?|bank\\s+ops|bank\\s+operations?|payment|payments?|platezh|oplata)`, "iu").test(lowered);
}
function extractCounterpartyFromFreeTextHeuristic(text) {
if (!hasDocsOrBankSignal(text)) {
@@ -413,14 +495,16 @@ function extractCounterpartyFromFreeTextHeuristic(text) {
}
function extractImplicitCounterpartyValue(text) {
const input = String(text ?? "");
const beforeDocsMatch = input.match(/(?:^|\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\s+(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)(?=[\s,.;:!?)]|$)/iu);
const beforeDocsPattern = new RegExp(`(?:^|\\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\\s+${DOC_SIGNAL_PATTERN}(?=[\\s,.;:!?)]|$)`, "iu");
const beforeDocsMatch = input.match(beforeDocsPattern);
if (beforeDocsMatch) {
const candidate = String(beforeDocsMatch[1] ?? "").trim();
if (isLikelyCounterpartyToken(candidate)) {
return candidate;
}
}
const afterDocsMatch = input.match(/(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)\s+(?:по\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\s,.;:!?)]|$)/iu);
const afterDocsPattern = new RegExp(`${DOC_SIGNAL_PATTERN}\\s+(?:по\\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\\s,.;:!?)]|$)`, "iu");
const afterDocsMatch = input.match(afterDocsPattern);
if (afterDocsMatch) {
const candidate = String(afterDocsMatch[1] ?? "").trim();
if (isLikelyCounterpartyToken(candidate)) {
@@ -444,7 +528,8 @@ function requiredFiltersByIntent(intent) {
return [];
}
function extractAddressFilters(userMessage, intent) {
const text = String(userMessage ?? "").trim();
const rawText = String(userMessage ?? "").trim();
const text = normalizeMojibakeString(rawText);
const filters = {
sort: "period_desc",
limit: 20
@@ -12,12 +12,17 @@ const ADDRESS_ACTION_TOKENS = [
"debt",
"owe",
"покажи",
"покаж",
"показ",
"список",
"найди",
"найд",
"выведи",
"вывед",
"кто",
"кому",
"какие",
"что по",
"остаток",
"долг",
"задолж",
@@ -70,6 +75,10 @@ const ADDRESS_ENTITY_TOKENS = [
"кредитор",
"аванс",
"оплат",
"поступлен",
"поступлени",
"списан",
"списани",
"долг",
"должен",
"должны",
@@ -138,6 +147,79 @@ function hasAddressFollowupSignal(text) {
}
return false;
}
function hasDocsOrBankSignal(text) {
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|поступлен|списан|transactions?|bank\s+ops|bank\s+operations?)/iu.test(text);
}
function hasLikelyCounterpartyToken(text) {
const stopWords = new Set([
"за",
"с",
"по",
"на",
"и",
"или",
"док",
"доки",
"документ",
"документы",
"документов",
"банк",
"банковские",
"операции",
"платежи",
"платеж",
"платёж",
"контрагент",
"контрагенту",
"контрагента",
"компания",
"компании",
"организация",
"организации",
"год",
"года",
"г",
"плс",
"pls",
"пж",
"пжлст",
"пожалуйста",
"бля",
"блять",
"епт",
"ёпт",
"епта",
"нах",
"нахуй",
"покеж",
"покажи",
"показать",
"покаж",
"выведи",
"show",
"list",
"please",
"all",
"vse"
]);
const tokens = String(text ?? "")
.split(/[^a-zа-яё0-9._-]+/iu)
.map((token) => token.trim())
.filter((token) => token.length >= 2);
return tokens.some((token) => {
const lowered = token.toLowerCase();
if (stopWords.has(lowered)) {
return false;
}
if (/^\d+$/.test(lowered)) {
return false;
}
if (/^(?:19|20)\d{2}$/.test(lowered)) {
return false;
}
return true;
});
}
function hasAnyToken(text, tokens) {
return tokens.some((token) => text.includes(token));
}
@@ -176,6 +258,13 @@ function detectAddressQuestionMode(userMessage) {
reasons: ["address_entity_detected"]
};
}
if (!hasDeepReasoning && hasDocsOrBankSignal(text) && (hasLooseByAnchor || hasLikelyCounterpartyToken(text))) {
return {
mode: "address_query",
confidence: "medium",
reasons: ["docs_or_bank_signal_detected", "anchor_like_token_detected"]
};
}
if (hasDeepReasoning) {
return {
mode: "deep_analysis",
@@ -18,7 +18,11 @@ function toNonEmptyString(value) {
return normalized.length > 0 ? normalized : null;
}
function hasAllTimeHint(text) {
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(String(text ?? ""));
const normalized = String(text ?? "");
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+весь\s+срок|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|за\s+любой\s+срок|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(normalized);
}
function hasSameDateHint(text) {
return /(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|та\s+же\s+дата|same\s+date|as\s+of\s+same\s+date|the\s+same\s+date)/iu.test(String(text ?? ""));
}
function hasAddressFollowupContextSignal(text) {
const normalized = String(text ?? "").trim();
@@ -28,10 +32,18 @@ function hasAddressFollowupContextSignal(text) {
if (hasAllTimeHint(normalized)) {
return true;
}
if (/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that)/iu.test(normalized)) {
if (/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|а\s+теперь|теперь|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that|then|now)/iu.test(normalized)) {
return true;
}
return normalized.split(/\s+/).filter(Boolean).length <= 8;
if (hasSameDateHint(normalized)) {
return true;
}
const tokenCount = normalized.split(/\s+/).filter(Boolean).length;
const hasPeriodLiteral = /\b(?:19|20)\d{2}(?:[./-](?:0?[1-9]|1[0-2]))?\b/.test(normalized);
if (tokenCount <= 8 && hasPeriodLiteral) {
return true;
}
return tokenCount <= 6;
}
function mergeFollowupFilters(current, intent, userMessage, followupContext) {
const merged = { ...current };
@@ -44,7 +56,11 @@ function mergeFollowupFilters(current, intent, userMessage, followupContext) {
const previousCounterparty = toNonEmptyString(previous.counterparty);
const previousContract = toNonEmptyString(previous.contract);
const previousAccount = toNonEmptyString(previous.account);
const previousAsOfDate = toNonEmptyString(previous.as_of_date);
const previousPeriodFrom = toNonEmptyString(previous.period_from);
const previousPeriodTo = toNonEmptyString(previous.period_to);
const allTimeRequested = hasAllTimeHint(userMessage);
const sameDateRequested = hasSameDateHint(userMessage);
if (intent === "list_documents_by_counterparty" || intent === "bank_operations_by_counterparty") {
if (!toNonEmptyString(merged.counterparty)) {
const inheritedCounterparty = previousCounterparty ??
@@ -57,18 +73,23 @@ function mergeFollowupFilters(current, intent, userMessage, followupContext) {
}
if (intent === "account_balance_snapshot" || intent === "documents_forming_balance") {
if (!toNonEmptyString(merged.account)) {
const inheritedAccount = previousAccount ??
(followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
const inheritedAccount = previousAccount ?? (followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
if (inheritedAccount) {
merged.account = inheritedAccount;
reasons.push("account_from_followup_context");
}
}
if (sameDateRequested) {
const inheritedAsOfDate = previousAsOfDate ?? previousPeriodTo ?? previousPeriodFrom;
if (inheritedAsOfDate && merged.as_of_date !== inheritedAsOfDate) {
merged.as_of_date = inheritedAsOfDate;
reasons.push("as_of_date_from_followup_context");
}
}
}
if (intent === "open_items_by_counterparty_or_contract" || intent === "list_open_contracts") {
if (!toNonEmptyString(merged.contract)) {
const inheritedContract = previousContract ??
(followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
const inheritedContract = previousContract ?? (followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
if (inheritedContract) {
merged.contract = inheritedContract;
reasons.push("contract_from_followup_context");
@@ -94,11 +115,11 @@ function mergeFollowupFilters(current, intent, userMessage, followupContext) {
const currentHasPeriod = hasExplicitPeriodWindow(merged);
const previousHasPeriod = hasExplicitPeriodWindow(previous);
if (!currentHasPeriod && previousHasPeriod && hasAddressFollowupContextSignal(userMessage)) {
if (toNonEmptyString(previous.period_from)) {
merged.period_from = previous.period_from;
if (previousPeriodFrom) {
merged.period_from = previousPeriodFrom;
}
if (toNonEmptyString(previous.period_to)) {
merged.period_to = previous.period_to;
if (previousPeriodTo) {
merged.period_to = previousPeriodTo;
}
reasons.push("period_from_followup_context");
}
+188 -14
View File
@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.AssistantService = void 0;
exports.evaluateCoverageForTests = evaluateCoverageForTests;
@@ -51,6 +54,7 @@ const companyAnchorResolver_1 = __importStar(require("./companyAnchorResolver"))
const assistantRuntimeGuards_1 = __importStar(require("./assistantRuntimeGuards"));
const assistantClaimBoundEvidence_1 = __importStar(require("./assistantClaimBoundEvidence"));
const addressQueryService_1 = __importStar(require("./addressQueryService"));
const iconv_lite_1 = __importDefault(require("iconv-lite"));
function retrievalSummaryForRoute(route) {
if (route === "store_canonical")
return "Canonical accounting data path selected.";
@@ -1424,7 +1428,7 @@ function hasAccountingSignal(text) {
}
function hasFollowupMarker(text) {
const compact = compactWhitespace(text.toLowerCase());
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|plus|also|dobav|utochn|prodolzh)/i.test(compact);
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|а теперь|теперь|plus|also|dobav|utochn|prodolzh|then|now)/i.test(compact);
}
function hasReferentialPointer(text) {
return /(по этому|по тому|это же|этой|этим|этому|из этого|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
@@ -1818,6 +1822,153 @@ function toNonEmptyString(value) {
const text = String(value).trim();
return text.length > 0 ? text : null;
}
const ADDRESS_PREDECOMPOSE_NOISE_TOKENS = new Set([
"за",
"с",
"по",
"на",
"и",
"или",
"док",
"доки",
"docs",
"documents",
"doki",
"dokument",
"dokumenty",
"документ",
"документы",
"документов",
"банк",
"банковские",
"операции",
"платеж",
"платёж",
"платежи",
"контрагент",
"контрагенту",
"контрагента",
"год",
"года",
"г",
"year",
"god",
"плс",
"pls",
"пж",
"пжлст",
"пожалуйста",
"please",
"покеж",
"покажи",
"показать",
"show",
"list",
"выведи",
"которые",
"какие",
"какой",
"есть",
"est",
"kakie",
"kakoi",
"vse",
"all",
"blya",
"blyat",
"епт",
"ёпт",
"бля"
]);
function textMojibakeScoreForAddress(value) {
const source = String(value ?? "");
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
}
function looksLikeMojibakeForAddress(value) {
const source = String(value ?? "");
if (!source.trim()) {
return false;
}
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
return true;
}
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
}
function repairAddressMojibake(value) {
const source = String(value ?? "");
if (!looksLikeMojibakeForAddress(source)) {
return source;
}
let candidate = source;
try {
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
candidate = fromWin1251;
}
}
catch (_error) { }
try {
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
if (textMojibakeScoreForAddress(fromLatin1) > textMojibakeScoreForAddress(candidate)) {
candidate = fromLatin1;
}
}
catch (_error) { }
return candidate;
}
function extractAddressAnchorTokens(value) {
const source = repairAddressMojibake(compactWhitespace(String(value ?? "").toLowerCase()));
if (!source) {
return [];
}
const tokens = source
.split(/[^a-zа-яё0-9._-]+/iu)
.map((item) => item.trim())
.filter((item) => item.length >= 2);
const filtered = [];
for (const token of tokens) {
if (/^\d+$/.test(token)) {
continue;
}
if (/^(?:19|20)\d{2}$/.test(token)) {
continue;
}
if (/^(?:0?[1-9]|1[0-2])[./-](?:19|20)\d{2}$/.test(token) || /^(?:19|20)\d{2}[./-](?:0?[1-9]|1[0-2])$/.test(token)) {
continue;
}
if (/^(?:янв|фев|мар|апр|май|июн|июл|авг|сен|сент|окт|ноя|дек|january|february|march|april|may|june|july|august|september|october|november|december)/i.test(token)) {
continue;
}
if (ADDRESS_PREDECOMPOSE_NOISE_TOKENS.has(token)) {
continue;
}
filtered.push(token);
}
return Array.from(new Set(filtered));
}
function selectPreferredAddressFragmentCandidate(rawText, normalizedText) {
const normalizedCandidate = compactWhitespace(repairAddressMojibake(normalizedText ?? ""));
const rawCandidate = compactWhitespace(repairAddressMojibake(rawText ?? ""));
if (!normalizedCandidate && !rawCandidate) {
return null;
}
if (!normalizedCandidate) {
return rawCandidate;
}
if (!rawCandidate) {
return normalizedCandidate;
}
const normalizedAnchors = extractAddressAnchorTokens(normalizedCandidate);
const rawAnchors = extractAddressAnchorTokens(rawCandidate);
if (rawAnchors.length > 0 && normalizedAnchors.length === 0) {
return rawCandidate;
}
return normalizedCandidate;
}
function readAddressFilterString(addressDebug, key) {
const filters = addressDebug?.extracted_filters;
if (!filters || typeof filters !== "object") {
@@ -1839,7 +1990,8 @@ function findLastAddressAssistantDebug(items) {
return null;
}
function hasAddressFollowupContextSignal(userMessage) {
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
const repaired = repairAddressMojibake(String(userMessage ?? ""));
const text = compactWhitespace(repaired.toLowerCase());
if (!text) {
return false;
}
@@ -1849,10 +2001,16 @@ function hasAddressFollowupContextSignal(userMessage) {
if (hasReferentialPointer(text)) {
return true;
}
if (/(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|same\s+date|the\s+same\s+date|as\s+of\s+same\s+date)/iu.test(text)) {
return true;
}
const shortFollowup = countTokens(text) <= 8;
if (shortFollowup && hasFollowupMarker(text)) {
return true;
}
if (shortFollowup && hasPeriodLiteral(text)) {
return true;
}
return false;
}
function resolveAddressFollowupCarryoverContext(userMessage, items) {
@@ -1889,11 +2047,12 @@ function resolveAddressFollowupCarryoverContext(userMessage, items) {
};
}
function isAddressLlmPreDecomposeCandidate(userMessage) {
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
const repaired = repairAddressMojibake(String(userMessage ?? ""));
const text = compactWhitespace(repaired.toLowerCase());
if (!text) {
return false;
}
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|банк|выписк|платеж|оплат|поступлен|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?)/i.test(text);
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|сальдо|банк|выписк|платеж|оплат|поступлен|поступлени|списан|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?|doki|dokument(?:y|ov|am|a)?|platezh|oplata|schet|saldo)/i.test(text);
}
function extractAddressQuestionFromNormalized(normalized) {
if (!normalized || typeof normalized !== "object") {
@@ -1911,13 +2070,16 @@ function extractAddressQuestionFromNormalized(normalized) {
continue;
}
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
if (readiness === "no_route") {
continue;
}
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
const rawText = toNonEmptyString(fragment.raw_fragment_text);
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
if (!candidate) {
continue;
}
if (candidate.length >= 3 && candidate.length <= 500) {
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
continue;
}
return candidate;
}
}
@@ -2024,13 +2186,16 @@ function extractAddressQuestionFromRawNormalizerOutput(rawModelOutput) {
continue;
}
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
if (readiness === "no_route") {
continue;
}
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
const rawText = toNonEmptyString(fragment.raw_fragment_text);
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
if (!candidate) {
continue;
}
if (candidate.length >= 3 && candidate.length <= 500) {
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
continue;
}
return candidate;
}
}
@@ -2241,12 +2406,21 @@ class AssistantService {
reason: "disabled_by_feature_flag"
};
const addressInputMessage = toNonEmptyString(addressPreDecompose?.effectiveMessage) ?? userMessage;
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
const shouldPreferContextualLane = Boolean(carryover?.followupContext);
if (shouldPreferContextualLane) {
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
followupContext: carryover.followupContext
});
if (contextualAddressLane?.handled) {
return finalizeAddressLaneResponse(contextualAddressLane, addressInputMessage, carryover, addressPreDecompose);
}
}
const primaryAddressLane = await this.addressQueryService.tryHandle(addressInputMessage);
if (primaryAddressLane?.handled) {
return finalizeAddressLaneResponse(primaryAddressLane, addressInputMessage, null, addressPreDecompose);
}
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
if (carryover?.followupContext) {
if (!shouldPreferContextualLane && carryover?.followupContext) {
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
followupContext: carryover.followupContext
});
@@ -11,6 +11,24 @@ const RETRY_INSTRUCTION_V1 = "IMPORTANT: return valid JSON strictly matching sch
const RETRY_INSTRUCTION_V2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2. No markdown.";
const RETRY_INSTRUCTION_V2_0_1 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_1. No markdown.";
const RETRY_INSTRUCTION_V2_0_2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_2. No markdown.";
const CONFIDENCE_LEVELS = ["high", "medium", "low"];
const DOMAIN_RELEVANCE_VALUES = ["in_scope", "out_of_scope", "unclear"];
const BUSINESS_SCOPE_VALUES = [
"company_specific_accounting",
"generic_accounting",
"offtopic",
"unclear"
];
const CANDIDATE_LABEL_VALUES = [
"heavy_analytical",
"cross_entity",
"drilldown_explain",
"rule_based_account_control",
"anomaly_probe",
"period_close_risk",
"ambiguous_human_query",
"simple_factual"
];
function safeJsonParse(text) {
const cleaned = text.trim().replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
return JSON.parse(cleaned);
@@ -58,6 +76,374 @@ function computeRetryMaxOutputTokens(current, rawModelResponse) {
const escalated = Math.max(current + 400, Math.ceil(current * 1.6));
return Math.min(escalated, 2400);
}
function normalizeToken(value) {
return String(value ?? "")
.trim()
.toLowerCase()
.replace(/[\s-]+/g, "_");
}
function toOptionalString(value) {
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function coerceBoolean(value, fallback = false) {
if (typeof value === "boolean") {
return value;
}
if (typeof value === "number") {
if (value === 1)
return true;
if (value === 0)
return false;
return fallback;
}
if (typeof value === "string") {
const token = value.trim().toLowerCase();
if (["true", "1", "yes", "y", "да", "ok"].includes(token)) {
return true;
}
if (["false", "0", "no", "n", "нет"].includes(token)) {
return false;
}
}
return fallback;
}
function coerceStringArray(value) {
if (Array.isArray(value)) {
return Array.from(new Set(value
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter((item) => item.length > 0)));
}
if (typeof value === "string") {
return Array.from(new Set(value
.split(/[,\n;]+/)
.map((item) => item.trim())
.filter((item) => item.length > 0)));
}
return [];
}
function coerceConfidence(value, fallback) {
if (typeof value === "string") {
const token = normalizeToken(value);
if (CONFIDENCE_LEVELS.includes(token)) {
return token;
}
}
if (typeof value === "number" && Number.isFinite(value)) {
const normalized = value > 1 ? value / 100 : value;
if (normalized >= 0.75)
return "high";
if (normalized >= 0.45)
return "medium";
return "low";
}
return fallback;
}
function coerceDomainRelevance(value, fallback) {
if (typeof value === "boolean") {
return value ? "in_scope" : "out_of_scope";
}
const token = normalizeToken(value);
if (DOMAIN_RELEVANCE_VALUES.includes(token)) {
return token;
}
if (["in_scope_true", "in_scope_yes", "in_scope_relevant", "relevant", "supported"].includes(token)) {
return "in_scope";
}
if (["out_scope", "outofscope", "offtopic", "off_topic", "irrelevant"].includes(token)) {
return "out_of_scope";
}
if (token === "true")
return "in_scope";
if (token === "false")
return "out_of_scope";
if (["unknown", "ambiguous", "n_a", "na"].includes(token))
return "unclear";
return fallback;
}
function coerceBusinessScope(value, fallback, domainRelevance) {
const token = normalizeToken(value);
if (BUSINESS_SCOPE_VALUES.includes(token)) {
return token;
}
if (["company_specific", "company_accounting", "document_review", "settlement", "bank_settlement"].includes(token)) {
return "company_specific_accounting";
}
if (["generic", "general_accounting", "general"].includes(token)) {
return "generic_accounting";
}
if (["out_of_scope", "off_topic", "outside"].includes(token)) {
return "offtopic";
}
if (token === "unknown") {
return "unclear";
}
if (domainRelevance === "out_of_scope") {
return "offtopic";
}
if (domainRelevance === "in_scope") {
return "company_specific_accounting";
}
return fallback;
}
function coerceFragmentId(value, index, fallback) {
if (typeof value === "string" && value.trim().length > 0) {
return value.trim();
}
if (typeof value === "number" && Number.isFinite(value)) {
const n = Math.max(1, Math.floor(value));
return `F${n}`;
}
return fallback || `F${index + 1}`;
}
function parseYear(value) {
if (typeof value === "number" && Number.isInteger(value) && value >= 1900 && value <= 2200) {
return value;
}
if (typeof value === "string") {
const trimmed = value.trim();
if (/^\d{4}$/.test(trimmed)) {
const parsed = Number.parseInt(trimmed, 10);
if (Number.isInteger(parsed) && parsed >= 1900 && parsed <= 2200) {
return parsed;
}
}
}
return null;
}
function parseMonth(value) {
if (typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 12) {
return value;
}
if (typeof value === "string") {
const trimmed = value.trim();
if (/^\d{1,2}$/.test(trimmed)) {
const parsed = Number.parseInt(trimmed, 10);
if (parsed >= 1 && parsed <= 12) {
return parsed;
}
}
}
return null;
}
function coerceTimeScope(value, rawText, fallback) {
if (value && typeof value === "object") {
const source = value;
const rawType = normalizeToken(source.type);
const confidence = coerceConfidence(source.confidence, fallback.confidence);
if (["explicit", "inferred", "missing"].includes(rawType)) {
if (rawType === "missing") {
return {
type: "missing",
value: null,
confidence
};
}
return {
type: rawType,
value: toOptionalString(source.value),
confidence
};
}
const periodType = normalizeToken(source.period_type);
const year = parseYear(source.year);
const month = parseMonth(source.month);
if ((periodType === "year" || (periodType.length === 0 && year !== null)) && year !== null) {
return {
type: "explicit",
value: String(year),
confidence: confidence === "low" ? "medium" : confidence
};
}
if ((periodType === "month" || periodType === "year_month" || (year !== null && month !== null)) && year !== null && month !== null) {
return {
type: "explicit",
value: `${year}-${String(month).padStart(2, "0")}`,
confidence: confidence === "low" ? "medium" : confidence
};
}
}
const inferred = inferTimeScope(rawText);
if (inferred.type !== "missing") {
return inferred;
}
return fallback;
}
function coerceFlags(value, fallback) {
if (!value || typeof value !== "object") {
return fallback;
}
const source = value;
const pick = (key, aliases = []) => {
if (key in source) {
return coerceBoolean(source[key], fallback[key]);
}
for (const alias of aliases) {
if (alias in source) {
return coerceBoolean(source[alias], fallback[key]);
}
}
return fallback[key];
};
return {
has_multi_entity_scope: pick("has_multi_entity_scope", ["multi_entity_scope"]),
asks_for_chain_explanation: pick("asks_for_chain_explanation", ["asks_for_chain", "chain_explanation"]),
asks_for_ranking_or_top: pick("asks_for_ranking_or_top", ["asks_for_ranking", "asks_for_top"]),
asks_for_period_summary: pick("asks_for_period_summary", ["period_summary"]),
asks_for_rule_check: pick("asks_for_rule_check", ["rule_check"]),
asks_for_anomaly_scan: pick("asks_for_anomaly_scan", ["anomaly_scan"]),
asks_for_exact_object_trace: pick("asks_for_exact_object_trace", ["exact_object_trace"]),
asks_for_evidence: pick("asks_for_evidence", ["evidence"]),
mentions_period_close_context: pick("mentions_period_close_context", ["period_close_context"])
};
}
function mapCandidateLabel(value) {
const token = normalizeToken(value);
if (CANDIDATE_LABEL_VALUES.includes(token)) {
return token;
}
if (["show_documents", "document_list", "show_docs", "point_answer", "lookup"].includes(token)) {
return "simple_factual";
}
if (["ranking", "top", "summary", "analytical"].includes(token)) {
return "heavy_analytical";
}
if (["chain", "cross", "cross_domain"].includes(token)) {
return "cross_entity";
}
if (["rule_check", "control", "rules"].includes(token)) {
return "rule_based_account_control";
}
if (["risk_scan", "anomaly", "risk"].includes(token)) {
return "anomaly_probe";
}
if (["period_close", "month_close"].includes(token)) {
return "period_close_risk";
}
if (["ambiguous", "unclear"].includes(token)) {
return "ambiguous_human_query";
}
return null;
}
function coerceCandidateLabels(value, flags, domainRelevance, fallback) {
const parsed = coerceStringArray(value)
.map((item) => mapCandidateLabel(item))
.filter((item) => Boolean(item));
if (parsed.length > 0) {
return Array.from(new Set(parsed));
}
const inferred = pickCandidateLabels(flags, domainRelevance);
if (inferred.length > 0) {
return inferred;
}
return fallback;
}
function coerceFragmentV2(rawFragment, index, userMessage) {
const source = rawFragment && typeof rawFragment === "object" ? rawFragment : {};
const rawText = toOptionalString(source.raw_fragment_text) ??
toOptionalString(source.rawText) ??
toOptionalString(source.fragment_text) ??
toOptionalString(source.text) ??
userMessage.trim();
const base = buildFragmentV2(rawText, index) ?? buildFragmentV2(userMessage, index);
if (!base) {
return null;
}
const domainRelevance = coerceDomainRelevance(source.domain_relevance, base.domain_relevance);
const businessScope = coerceBusinessScope(source.business_scope, base.business_scope, domainRelevance);
const flags = coerceFlags(source.flags, base.flags);
const entityHints = coerceStringArray(source.entity_hints);
const accountHints = coerceStringArray(source.account_hints);
const documentHints = coerceStringArray(source.document_hints);
const registerHints = coerceStringArray(source.register_hints);
return {
fragment_id: coerceFragmentId(source.fragment_id, index, base.fragment_id),
raw_fragment_text: rawText,
normalized_fragment_text: toOptionalString(source.normalized_fragment_text) ?? base.normalized_fragment_text,
domain_relevance: domainRelevance,
business_scope: businessScope,
entity_hints: entityHints.length > 0 ? entityHints : base.entity_hints,
account_hints: accountHints.length > 0 ? accountHints : base.account_hints,
document_hints: documentHints.length > 0 ? documentHints : base.document_hints,
register_hints: registerHints.length > 0 ? registerHints : base.register_hints,
time_scope: coerceTimeScope(source.time_scope, rawText, base.time_scope),
flags,
candidate_labels: coerceCandidateLabels(source.candidate_labels, flags, domainRelevance, base.candidate_labels),
confidence: coerceConfidence(source.confidence, base.confidence)
};
}
function coerceDiscardedFragments(value) {
if (!Array.isArray(value)) {
return [];
}
const collected = [];
for (const item of value) {
if (!item || typeof item !== "object") {
continue;
}
const source = item;
const raw = toOptionalString(source.raw_fragment_text);
const reason = toOptionalString(source.reason);
if (!raw || !reason) {
continue;
}
collected.push({
raw_fragment_text: raw,
reason
});
}
return collected;
}
function coerceScopeConfidence(value, fallback) {
return coerceConfidence(value, fallback);
}
function coerceGlobalNotes(value, fallbackNeedsClarification) {
if (!value || typeof value !== "object") {
return {
needs_clarification: fallbackNeedsClarification,
clarification_reason: fallbackNeedsClarification ? "clarification_required" : null
};
}
const source = value;
const needs = coerceBoolean(source.needs_clarification, fallbackNeedsClarification);
const clarificationReason = toOptionalString(source.clarification_reason);
return {
needs_clarification: needs,
clarification_reason: needs ? clarificationReason ?? "clarification_required" : null
};
}
function coerceNormalizedCandidateV2(candidate, userMessage) {
if (!candidate || typeof candidate !== "object") {
return null;
}
const source = candidate;
const sourceFragments = Array.isArray(source.fragments)
? source.fragments
: source.fragment && typeof source.fragment === "object"
? [source.fragment]
: splitIntoCandidateFragments(userMessage).map((text) => ({ raw_fragment_text: text }));
const fragments = sourceFragments
.map((item, index) => coerceFragmentV2(item, index, userMessage))
.filter((item) => item !== null);
const inScopeCount = fragments.filter((item) => item.domain_relevance === "in_scope").length;
const unclearCount = fragments.filter((item) => item.domain_relevance === "unclear").length;
const messageInScope = inScopeCount > 0;
const inferredScopeConfidence = messageInScope ? (unclearCount > 0 ? "medium" : "high") : "low";
const inferredNeedsClarification = messageInScope && (unclearCount > 0 || fragments.some((item) => item.time_scope.type === "missing"));
return {
schema_version: "normalized_query_v2",
user_message_raw: toOptionalString(source.user_message_raw) ?? userMessage,
message_in_scope: coerceBoolean(source.message_in_scope, messageInScope),
scope_confidence: coerceScopeConfidence(source.scope_confidence, inferredScopeConfidence),
contains_multiple_tasks: coerceBoolean(source.contains_multiple_tasks, fragments.length > 1),
fragments,
discarded_fragments: coerceDiscardedFragments(source.discarded_fragments),
global_notes: coerceGlobalNotes(source.global_notes, inferredNeedsClarification)
};
}
function collectDateSpans(text) {
const spans = [];
const patterns = [
@@ -927,6 +1313,7 @@ class NormalizerService {
try {
normalizedCandidate = safeJsonParse(outputText);
if (schemaVersion !== "v1") {
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
}
if (schemaVersion === "v2_0_2") {
@@ -975,6 +1362,7 @@ class NormalizerService {
try {
normalizedCandidate = safeJsonParse(outputText);
if (schemaVersion !== "v1") {
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
}
if (schemaVersion === "v2_0_2") {