АДРЕСНЫЙ РЕЖИМ - ллм декомпоз

This commit is contained in:
2026-04-01 22:11:40 +03:00
parent 18e0f1364d
commit b2d32f869c
1540 changed files with 10464 additions and 1451 deletions
@@ -1,4 +1,5 @@
import type { AddressFilterExtraction, AddressFilterSet, AddressIntent } from "../types/addressQuery";
import iconv from "iconv-lite";
const ACCOUNT_PATTERN = /(?:сч[её]т|счет|account)[^0-9]{0,12}(\d{2}(?:[.,]\d{1,2})?)/i;
const LIMIT_PATTERN = /(?:\btop\b|\blimit\b|\bпервые\b|\bтоп\b)\s*(\d{1,3})/i;
@@ -14,8 +15,10 @@ const YEAR_RANGE_PATTERN =
/(?:за|for|с|from)?\s*(20\d{2})\s*(?:[-]|до|to|по)\s*(20\d{2})(?:\s*(?:г(?:од|ода)?\.?|year))?(?=[^\d]|$)/iu;
const YEAR_RANGE_LOOSE_PATTERN = /\b(20\d{2})\b\s*(?:[-]|до|to|по)\s*\b(20\d{2})\b/iu;
const YEAR_PERIOD_PATTERN =
/(?:за|for)\s*(20\d{2})(?!\s*(?:[-]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year)?/iu;
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year)(?=$|[\s,.;:!?()\-])/iu;
/(?:за|for)\s*(20\d{2})(?!\s*(?:[-]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year|god)?/iu;
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year|god)(?=$|[\s,.;:!?()\-])/iu;
const YEAR_PERIOD_SHORT_ORDINAL_PATTERN =
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(\d{2})\s*(?:[-\s]?(?:й|ый|ой|th))(?:\s*(?:г(?:од|ода)?\.?|year|period|период))?(?=$|[\s,.;:!?()\-])/iu;
const YEAR_PERIOD_ANY_PATTERN =
/(?:^|[\s,.;:!?()\-])((?:19|20)\d{2})(?!\s*(?:[-]|до|to|по)\s*(?:19|20)\d{2})(?![.\/-]\d)(?:\s*(?:г(?:од|ода)?\.?|year))?(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NUMERIC_MONTH_YEAR_PATTERN =
@@ -26,6 +29,58 @@ const MONTH_PERIOD_NAME_PATTERN =
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*([a-zа-яё]+)\s+(20\d{2})(?:\s*г(?:од|ода|\\.)?)?(?=$|[\s,.;:!?()\-])/iu;
const MONTH_PERIOD_NAME_YEAR_FIRST_PATTERN =
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(20\d{2})(?:\s*г(?:од|ода|\\.)?)?\s+([a-zа-яё]+)(?=$|[\s,.;:!?()\-])/iu;
const DOC_SIGNAL_PATTERN =
"(?:док(?:и|умент|ументы|ументов|умам|ума)|docs?|documents?|doki|dokument(?:y|ov|am|a)?)";
function textMojibakeScore(value: string): number {
const source = String(value ?? "");
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
}
function looksLikeMojibake(value: string): boolean {
const source = String(value ?? "");
if (!source.trim()) {
return false;
}
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
return true;
}
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
}
function decodeUtf8FromWin1251Mojibake(value: string): string {
if (!looksLikeMojibake(value)) {
return value;
}
try {
const bytes = iconv.encode(value, "win1251");
const decoded = bytes.toString("utf8");
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
} catch {
return value;
}
}
function decodeUtf8FromLatin1Mojibake(value: string): string {
if (!looksLikeMojibake(value)) {
return value;
}
try {
const decoded = Buffer.from(value, "latin1").toString("utf8");
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
} catch {
return value;
}
}
function normalizeMojibakeString(value: string): string {
const fromWin1251 = decodeUtf8FromWin1251Mojibake(value);
return decodeUtf8FromLatin1Mojibake(fromWin1251);
}
function toIsoDate(year: number, month: number, day: number): string | null {
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
@@ -200,18 +255,30 @@ function extractYearPeriod(text: string): { period_from?: string; period_to?: st
}
const shortYearMatch = text.match(YEAR_PERIOD_SHORT_PATTERN);
if (!shortYearMatch) {
return {};
if (shortYearMatch) {
const shortYear = Number(shortYearMatch[1]);
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
}
}
const shortYear = Number(shortYearMatch[1]);
if (!Number.isFinite(shortYear) || shortYear < 0 || shortYear > 99) {
return {};
const shortOrdinalMatch = text.match(YEAR_PERIOD_SHORT_ORDINAL_PATTERN);
if (shortOrdinalMatch) {
const shortYear = Number(shortOrdinalMatch[1]);
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
}
}
const year = 2000 + shortYear;
return {
period_from: `${year}-01-01`,
period_to: `${year}-12-31`
};
return {};
}
function extractYearRangePeriod(text: string): { period_from?: string; period_to?: string } {
@@ -374,14 +441,36 @@ function isLikelyCounterpartyToken(rawToken: string): boolean {
"нахуй",
"покеж",
"покажи",
"выведи"
"показать",
"выведи",
"show",
"list",
"please",
"vse",
"all",
"kakie",
"kakoi",
"est",
"pokaji",
"pokazhi",
"pokazh",
"pokezh",
"doki",
"doky",
"dokument",
"dokumenty",
"documents",
"docs"
]);
return !stopWords.has(lowered);
}
function hasDocsOrBankSignal(text: string): boolean {
const lowered = String(text ?? "").toLowerCase();
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|transactions?|bank\s+ops|bank\s+operations?)/iu.test(
return new RegExp(
`(?:${DOC_SIGNAL_PATTERN}|банк|выписк|платеж|платёж|оплат|transactions?|bank\\s+ops|bank\\s+operations?|payment|payments?|platezh|oplata)`,
"iu"
).test(
lowered
);
}
@@ -448,9 +537,11 @@ function extractCounterpartyFromFreeTextHeuristic(text: string): string | undefi
function extractImplicitCounterpartyValue(text: string): string | undefined {
const input = String(text ?? "");
const beforeDocsMatch = input.match(
/(?:^|\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\s+(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)(?=[\s,.;:!?)]|$)/iu
const beforeDocsPattern = new RegExp(
`(?:^|\\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\\s+${DOC_SIGNAL_PATTERN}(?=[\\s,.;:!?)]|$)`,
"iu"
);
const beforeDocsMatch = input.match(beforeDocsPattern);
if (beforeDocsMatch) {
const candidate = String(beforeDocsMatch[1] ?? "").trim();
if (isLikelyCounterpartyToken(candidate)) {
@@ -458,9 +549,11 @@ function extractImplicitCounterpartyValue(text: string): string | undefined {
}
}
const afterDocsMatch = input.match(
/(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)\s+(?:по\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\s,.;:!?)]|$)/iu
const afterDocsPattern = new RegExp(
`${DOC_SIGNAL_PATTERN}\\s+(?:по\\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\\s,.;:!?)]|$)`,
"iu"
);
const afterDocsMatch = input.match(afterDocsPattern);
if (afterDocsMatch) {
const candidate = String(afterDocsMatch[1] ?? "").trim();
if (isLikelyCounterpartyToken(candidate)) {
@@ -488,7 +581,8 @@ function requiredFiltersByIntent(intent: AddressIntent): Array<keyof AddressFilt
}
export function extractAddressFilters(userMessage: string, intent: AddressIntent): AddressFilterExtraction {
const text = String(userMessage ?? "").trim();
const rawText = String(userMessage ?? "").trim();
const text = normalizeMojibakeString(rawText);
const filters: AddressFilterSet = {
sort: "period_desc",
limit: 20
@@ -628,3 +722,5 @@ export function extractAddressFilters(userMessage: string, intent: AddressIntent
warnings
};
}
@@ -11,12 +11,17 @@ const ADDRESS_ACTION_TOKENS = [
"debt",
"owe",
"покажи",
"покаж",
"показ",
"список",
"найди",
"найд",
"выведи",
"вывед",
"кто",
"кому",
"какие",
"что по",
"остаток",
"долг",
"задолж",
@@ -70,6 +75,10 @@ const ADDRESS_ENTITY_TOKENS = [
"кредитор",
"аванс",
"оплат",
"поступлен",
"поступлени",
"списан",
"списани",
"долг",
"должен",
"должны",
@@ -142,6 +151,83 @@ function hasAddressFollowupSignal(text: string): boolean {
return false;
}
function hasDocsOrBankSignal(text: string): boolean {
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|поступлен|списан|transactions?|bank\s+ops|bank\s+operations?)/iu.test(
text
);
}
function hasLikelyCounterpartyToken(text: string): boolean {
const stopWords = new Set([
"за",
"с",
"по",
"на",
"и",
"или",
"док",
"доки",
"документ",
"документы",
"документов",
"банк",
"банковские",
"операции",
"платежи",
"платеж",
"платёж",
"контрагент",
"контрагенту",
"контрагента",
"компания",
"компании",
"организация",
"организации",
"год",
"года",
"г",
"плс",
"pls",
"пж",
"пжлст",
"пожалуйста",
"бля",
"блять",
"епт",
"ёпт",
"епта",
"нах",
"нахуй",
"покеж",
"покажи",
"показать",
"покаж",
"выведи",
"show",
"list",
"please",
"all",
"vse"
]);
const tokens = String(text ?? "")
.split(/[^a-zа-яё0-9._-]+/iu)
.map((token) => token.trim())
.filter((token) => token.length >= 2);
return tokens.some((token) => {
const lowered = token.toLowerCase();
if (stopWords.has(lowered)) {
return false;
}
if (/^\d+$/.test(lowered)) {
return false;
}
if (/^(?:19|20)\d{2}$/.test(lowered)) {
return false;
}
return true;
});
}
function hasAnyToken(text: string, tokens: string[]): boolean {
return tokens.some((token) => text.includes(token));
}
@@ -186,6 +272,14 @@ export function detectAddressQuestionMode(userMessage: string): AddressModeDetec
};
}
if (!hasDeepReasoning && hasDocsOrBankSignal(text) && (hasLooseByAnchor || hasLikelyCounterpartyToken(text))) {
return {
mode: "address_query",
confidence: "medium",
reasons: ["docs_or_bank_signal_detected", "anchor_like_token_detected"]
};
}
if (hasDeepReasoning) {
return {
mode: "deep_analysis",
@@ -1,4 +1,4 @@
import type {
import type {
AddressFilterSet,
AddressIntent,
AddressIntentResolution,
@@ -45,7 +45,14 @@ function toNonEmptyString(value: unknown): string | null {
}
function hasAllTimeHint(text: string): boolean {
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(
const normalized = String(text ?? "");
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+весь\s+срок|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|за\s+любой\s+срок|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(
normalized
);
}
function hasSameDateHint(text: string): boolean {
return /(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|та\s+же\s+дата|same\s+date|as\s+of\s+same\s+date|the\s+same\s+date)/iu.test(
String(text ?? "")
);
}
@@ -58,10 +65,23 @@ export function hasAddressFollowupContextSignal(text: string): boolean {
if (hasAllTimeHint(normalized)) {
return true;
}
if (/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that)/iu.test(normalized)) {
if (
/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|а\s+теперь|теперь|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that|then|now)/iu.test(
normalized
)
) {
return true;
}
return normalized.split(/\s+/).filter(Boolean).length <= 8;
if (hasSameDateHint(normalized)) {
return true;
}
const tokenCount = normalized.split(/\s+/).filter(Boolean).length;
const hasPeriodLiteral = /\b(?:19|20)\d{2}(?:[./-](?:0?[1-9]|1[0-2]))?\b/.test(normalized);
if (tokenCount <= 8 && hasPeriodLiteral) {
return true;
}
return tokenCount <= 6;
}
function mergeFollowupFilters(
@@ -81,7 +101,11 @@ function mergeFollowupFilters(
const previousCounterparty = toNonEmptyString(previous.counterparty);
const previousContract = toNonEmptyString(previous.contract);
const previousAccount = toNonEmptyString(previous.account);
const previousAsOfDate = toNonEmptyString(previous.as_of_date);
const previousPeriodFrom = toNonEmptyString(previous.period_from);
const previousPeriodTo = toNonEmptyString(previous.period_to);
const allTimeRequested = hasAllTimeHint(userMessage);
const sameDateRequested = hasSameDateHint(userMessage);
if (intent === "list_documents_by_counterparty" || intent === "bank_operations_by_counterparty") {
if (!toNonEmptyString(merged.counterparty)) {
@@ -97,21 +121,24 @@ function mergeFollowupFilters(
if (intent === "account_balance_snapshot" || intent === "documents_forming_balance") {
if (!toNonEmptyString(merged.account)) {
const inheritedAccount =
previousAccount ??
(followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
const inheritedAccount = previousAccount ?? (followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
if (inheritedAccount) {
merged.account = inheritedAccount;
reasons.push("account_from_followup_context");
}
}
if (sameDateRequested) {
const inheritedAsOfDate = previousAsOfDate ?? previousPeriodTo ?? previousPeriodFrom;
if (inheritedAsOfDate && merged.as_of_date !== inheritedAsOfDate) {
merged.as_of_date = inheritedAsOfDate;
reasons.push("as_of_date_from_followup_context");
}
}
}
if (intent === "open_items_by_counterparty_or_contract" || intent === "list_open_contracts") {
if (!toNonEmptyString(merged.contract)) {
const inheritedContract =
previousContract ??
(followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
const inheritedContract = previousContract ?? (followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
if (inheritedContract) {
merged.contract = inheritedContract;
reasons.push("contract_from_followup_context");
@@ -140,11 +167,11 @@ function mergeFollowupFilters(
const currentHasPeriod = hasExplicitPeriodWindow(merged);
const previousHasPeriod = hasExplicitPeriodWindow(previous);
if (!currentHasPeriod && previousHasPeriod && hasAddressFollowupContextSignal(userMessage)) {
if (toNonEmptyString(previous.period_from)) {
merged.period_from = previous.period_from;
if (previousPeriodFrom) {
merged.period_from = previousPeriodFrom;
}
if (toNonEmptyString(previous.period_to)) {
merged.period_to = previous.period_to;
if (previousPeriodTo) {
merged.period_to = previousPeriodTo;
}
reasons.push("period_from_followup_context");
}
@@ -241,4 +268,3 @@ export function runAddressDecomposeStage(
baseReasons
};
}
@@ -13,6 +13,7 @@ import * as companyAnchorResolver_1 from "./companyAnchorResolver";
import * as assistantRuntimeGuards_1 from "./assistantRuntimeGuards";
import * as assistantClaimBoundEvidence_1 from "./assistantClaimBoundEvidence";
import * as addressQueryService_1 from "./addressQueryService";
import iconv from "iconv-lite";
function retrievalSummaryForRoute(route) {
if (route === "store_canonical")
return "Canonical accounting data path selected.";
@@ -1386,7 +1387,7 @@ function hasAccountingSignal(text) {
}
function hasFollowupMarker(text) {
const compact = compactWhitespace(text.toLowerCase());
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|plus|also|dobav|utochn|prodolzh)/i.test(compact);
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|а теперь|теперь|plus|also|dobav|utochn|prodolzh|then|now)/i.test(compact);
}
function hasReferentialPointer(text) {
return /(по этому|по тому|это же|этой|этим|этому|из этого|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
@@ -1780,6 +1781,153 @@ function toNonEmptyString(value) {
const text = String(value).trim();
return text.length > 0 ? text : null;
}
const ADDRESS_PREDECOMPOSE_NOISE_TOKENS = new Set([
"за",
"с",
"по",
"на",
"и",
"или",
"док",
"доки",
"docs",
"documents",
"doki",
"dokument",
"dokumenty",
"документ",
"документы",
"документов",
"банк",
"банковские",
"операции",
"платеж",
"платёж",
"платежи",
"контрагент",
"контрагенту",
"контрагента",
"год",
"года",
"г",
"year",
"god",
"плс",
"pls",
"пж",
"пжлст",
"пожалуйста",
"please",
"покеж",
"покажи",
"показать",
"show",
"list",
"выведи",
"которые",
"какие",
"какой",
"есть",
"est",
"kakie",
"kakoi",
"vse",
"all",
"blya",
"blyat",
"епт",
"ёпт",
"бля"
]);
function textMojibakeScoreForAddress(value) {
const source = String(value ?? "");
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
}
function looksLikeMojibakeForAddress(value) {
const source = String(value ?? "");
if (!source.trim()) {
return false;
}
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
return true;
}
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
}
function repairAddressMojibake(value) {
const source = String(value ?? "");
if (!looksLikeMojibakeForAddress(source)) {
return source;
}
let candidate = source;
try {
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
candidate = fromWin1251;
}
}
catch (_error) { }
try {
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
if (textMojibakeScoreForAddress(fromLatin1) > textMojibakeScoreForAddress(candidate)) {
candidate = fromLatin1;
}
}
catch (_error) { }
return candidate;
}
function extractAddressAnchorTokens(value) {
const source = repairAddressMojibake(compactWhitespace(String(value ?? "").toLowerCase()));
if (!source) {
return [];
}
const tokens = source
.split(/[^a-zа-яё0-9._-]+/iu)
.map((item) => item.trim())
.filter((item) => item.length >= 2);
const filtered = [];
for (const token of tokens) {
if (/^\d+$/.test(token)) {
continue;
}
if (/^(?:19|20)\d{2}$/.test(token)) {
continue;
}
if (/^(?:0?[1-9]|1[0-2])[./-](?:19|20)\d{2}$/.test(token) || /^(?:19|20)\d{2}[./-](?:0?[1-9]|1[0-2])$/.test(token)) {
continue;
}
if (/^(?:янв|фев|мар|апр|май|июн|июл|авг|сен|сент|окт|ноя|дек|january|february|march|april|may|june|july|august|september|october|november|december)/i.test(token)) {
continue;
}
if (ADDRESS_PREDECOMPOSE_NOISE_TOKENS.has(token)) {
continue;
}
filtered.push(token);
}
return Array.from(new Set(filtered));
}
function selectPreferredAddressFragmentCandidate(rawText, normalizedText) {
const normalizedCandidate = compactWhitespace(repairAddressMojibake(normalizedText ?? ""));
const rawCandidate = compactWhitespace(repairAddressMojibake(rawText ?? ""));
if (!normalizedCandidate && !rawCandidate) {
return null;
}
if (!normalizedCandidate) {
return rawCandidate;
}
if (!rawCandidate) {
return normalizedCandidate;
}
const normalizedAnchors = extractAddressAnchorTokens(normalizedCandidate);
const rawAnchors = extractAddressAnchorTokens(rawCandidate);
if (rawAnchors.length > 0 && normalizedAnchors.length === 0) {
return rawCandidate;
}
return normalizedCandidate;
}
function readAddressFilterString(addressDebug, key) {
const filters = addressDebug?.extracted_filters;
if (!filters || typeof filters !== "object") {
@@ -1801,7 +1949,8 @@ function findLastAddressAssistantDebug(items) {
return null;
}
function hasAddressFollowupContextSignal(userMessage) {
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
const repaired = repairAddressMojibake(String(userMessage ?? ""));
const text = compactWhitespace(repaired.toLowerCase());
if (!text) {
return false;
}
@@ -1811,10 +1960,16 @@ function hasAddressFollowupContextSignal(userMessage) {
if (hasReferentialPointer(text)) {
return true;
}
if (/(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|same\s+date|the\s+same\s+date|as\s+of\s+same\s+date)/iu.test(text)) {
return true;
}
const shortFollowup = countTokens(text) <= 8;
if (shortFollowup && hasFollowupMarker(text)) {
return true;
}
if (shortFollowup && hasPeriodLiteral(text)) {
return true;
}
return false;
}
function resolveAddressFollowupCarryoverContext(userMessage, items) {
@@ -1851,11 +2006,12 @@ function resolveAddressFollowupCarryoverContext(userMessage, items) {
};
}
function isAddressLlmPreDecomposeCandidate(userMessage) {
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
const repaired = repairAddressMojibake(String(userMessage ?? ""));
const text = compactWhitespace(repaired.toLowerCase());
if (!text) {
return false;
}
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|банк|выписк|платеж|оплат|поступлен|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?)/i.test(text);
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|сальдо|банк|выписк|платеж|оплат|поступлен|поступлени|списан|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?|doki|dokument(?:y|ov|am|a)?|platezh|oplata|schet|saldo)/i.test(text);
}
function extractAddressQuestionFromNormalized(normalized) {
if (!normalized || typeof normalized !== "object") {
@@ -1873,13 +2029,16 @@ function extractAddressQuestionFromNormalized(normalized) {
continue;
}
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
if (readiness === "no_route") {
continue;
}
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
const rawText = toNonEmptyString(fragment.raw_fragment_text);
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
if (!candidate) {
continue;
}
if (candidate.length >= 3 && candidate.length <= 500) {
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
continue;
}
return candidate;
}
}
@@ -1986,13 +2145,16 @@ function extractAddressQuestionFromRawNormalizerOutput(rawModelOutput) {
continue;
}
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
if (readiness === "no_route") {
continue;
}
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
const rawText = toNonEmptyString(fragment.raw_fragment_text);
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
if (!candidate) {
continue;
}
if (candidate.length >= 3 && candidate.length <= 500) {
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
continue;
}
return candidate;
}
}
@@ -2203,12 +2365,21 @@ export class AssistantService {
reason: "disabled_by_feature_flag"
};
const addressInputMessage = toNonEmptyString(addressPreDecompose?.effectiveMessage) ?? userMessage;
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
const shouldPreferContextualLane = Boolean(carryover?.followupContext);
if (shouldPreferContextualLane) {
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
followupContext: carryover.followupContext
});
if (contextualAddressLane?.handled) {
return finalizeAddressLaneResponse(contextualAddressLane, addressInputMessage, carryover, addressPreDecompose);
}
}
const primaryAddressLane = await this.addressQueryService.tryHandle(addressInputMessage);
if (primaryAddressLane?.handled) {
return finalizeAddressLaneResponse(primaryAddressLane, addressInputMessage, null, addressPreDecompose);
}
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
if (carryover?.followupContext) {
if (!shouldPreferContextualLane && carryover?.followupContext) {
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
followupContext: carryover.followupContext
});
@@ -6,6 +6,7 @@ import { toRouteHintSummary } from "./routeHintAdapter";
import { validateNormalized } from "./schemaValidator";
import { redactRequestPayload, saveEvalCase, saveTrace, type TraceRecord } from "./traceLogger";
import type {
DiscardedFragmentV2,
ExecutionReadiness,
NoRouteReason,
NormalizeRequestPayload,
@@ -27,6 +28,24 @@ const RETRY_INSTRUCTION_V1 = "IMPORTANT: return valid JSON strictly matching sch
const RETRY_INSTRUCTION_V2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2. No markdown.";
const RETRY_INSTRUCTION_V2_0_1 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_1. No markdown.";
const RETRY_INSTRUCTION_V2_0_2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_2. No markdown.";
const CONFIDENCE_LEVELS: ReadonlyArray<NormalizedFragmentV2["confidence"]> = ["high", "medium", "low"];
const DOMAIN_RELEVANCE_VALUES: ReadonlyArray<NormalizedFragmentV2["domain_relevance"]> = ["in_scope", "out_of_scope", "unclear"];
const BUSINESS_SCOPE_VALUES: ReadonlyArray<NormalizedFragmentV2["business_scope"]> = [
"company_specific_accounting",
"generic_accounting",
"offtopic",
"unclear"
];
const CANDIDATE_LABEL_VALUES: ReadonlyArray<NormalizedFragmentV2["candidate_labels"][number]> = [
"heavy_analytical",
"cross_entity",
"drilldown_explain",
"rule_based_account_control",
"anomaly_probe",
"period_close_risk",
"ambiguous_human_query",
"simple_factual"
];
function safeJsonParse(text: string): unknown {
const cleaned = text.trim().replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
@@ -79,6 +98,429 @@ function computeRetryMaxOutputTokens(current: number, rawModelResponse: unknown)
return Math.min(escalated, 2400);
}
function normalizeToken(value: unknown): string {
return String(value ?? "")
.trim()
.toLowerCase()
.replace(/[\s-]+/g, "_");
}
function toOptionalString(value: unknown): string | null {
if (typeof value !== "string") {
return null;
}
const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null;
}
function coerceBoolean(value: unknown, fallback = false): boolean {
if (typeof value === "boolean") {
return value;
}
if (typeof value === "number") {
if (value === 1) return true;
if (value === 0) return false;
return fallback;
}
if (typeof value === "string") {
const token = value.trim().toLowerCase();
if (["true", "1", "yes", "y", "да", "ok"].includes(token)) {
return true;
}
if (["false", "0", "no", "n", "нет"].includes(token)) {
return false;
}
}
return fallback;
}
function coerceStringArray(value: unknown): string[] {
if (Array.isArray(value)) {
return Array.from(
new Set(
value
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter((item) => item.length > 0)
)
);
}
if (typeof value === "string") {
return Array.from(
new Set(
value
.split(/[,\n;]+/)
.map((item) => item.trim())
.filter((item) => item.length > 0)
)
);
}
return [];
}
function coerceConfidence(value: unknown, fallback: NormalizedFragmentV2["confidence"]): NormalizedFragmentV2["confidence"] {
if (typeof value === "string") {
const token = normalizeToken(value);
if (CONFIDENCE_LEVELS.includes(token as NormalizedFragmentV2["confidence"])) {
return token as NormalizedFragmentV2["confidence"];
}
}
if (typeof value === "number" && Number.isFinite(value)) {
const normalized = value > 1 ? value / 100 : value;
if (normalized >= 0.75) return "high";
if (normalized >= 0.45) return "medium";
return "low";
}
return fallback;
}
function coerceDomainRelevance(
value: unknown,
fallback: NormalizedFragmentV2["domain_relevance"]
): NormalizedFragmentV2["domain_relevance"] {
if (typeof value === "boolean") {
return value ? "in_scope" : "out_of_scope";
}
const token = normalizeToken(value);
if (DOMAIN_RELEVANCE_VALUES.includes(token as NormalizedFragmentV2["domain_relevance"])) {
return token as NormalizedFragmentV2["domain_relevance"];
}
if (["in_scope_true", "in_scope_yes", "in_scope_relevant", "relevant", "supported"].includes(token)) {
return "in_scope";
}
if (["out_scope", "outofscope", "offtopic", "off_topic", "irrelevant"].includes(token)) {
return "out_of_scope";
}
if (token === "true") return "in_scope";
if (token === "false") return "out_of_scope";
if (["unknown", "ambiguous", "n_a", "na"].includes(token)) return "unclear";
return fallback;
}
function coerceBusinessScope(
value: unknown,
fallback: NormalizedFragmentV2["business_scope"],
domainRelevance: NormalizedFragmentV2["domain_relevance"]
): NormalizedFragmentV2["business_scope"] {
const token = normalizeToken(value);
if (BUSINESS_SCOPE_VALUES.includes(token as NormalizedFragmentV2["business_scope"])) {
return token as NormalizedFragmentV2["business_scope"];
}
if (["company_specific", "company_accounting", "document_review", "settlement", "bank_settlement"].includes(token)) {
return "company_specific_accounting";
}
if (["generic", "general_accounting", "general"].includes(token)) {
return "generic_accounting";
}
if (["out_of_scope", "off_topic", "outside"].includes(token)) {
return "offtopic";
}
if (token === "unknown") {
return "unclear";
}
if (domainRelevance === "out_of_scope") {
return "offtopic";
}
if (domainRelevance === "in_scope") {
return "company_specific_accounting";
}
return fallback;
}
function coerceFragmentId(value: unknown, index: number, fallback: string): string {
if (typeof value === "string" && value.trim().length > 0) {
return value.trim();
}
if (typeof value === "number" && Number.isFinite(value)) {
const n = Math.max(1, Math.floor(value));
return `F${n}`;
}
return fallback || `F${index + 1}`;
}
function parseYear(value: unknown): number | null {
if (typeof value === "number" && Number.isInteger(value) && value >= 1900 && value <= 2200) {
return value;
}
if (typeof value === "string") {
const trimmed = value.trim();
if (/^\d{4}$/.test(trimmed)) {
const parsed = Number.parseInt(trimmed, 10);
if (Number.isInteger(parsed) && parsed >= 1900 && parsed <= 2200) {
return parsed;
}
}
}
return null;
}
function parseMonth(value: unknown): number | null {
if (typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 12) {
return value;
}
if (typeof value === "string") {
const trimmed = value.trim();
if (/^\d{1,2}$/.test(trimmed)) {
const parsed = Number.parseInt(trimmed, 10);
if (parsed >= 1 && parsed <= 12) {
return parsed;
}
}
}
return null;
}
function coerceTimeScope(
value: unknown,
rawText: string,
fallback: NormalizedFragmentV2["time_scope"]
): NormalizedFragmentV2["time_scope"] {
if (value && typeof value === "object") {
const source = value as Record<string, unknown>;
const rawType = normalizeToken(source.type);
const confidence = coerceConfidence(source.confidence, fallback.confidence);
if (["explicit", "inferred", "missing"].includes(rawType)) {
if (rawType === "missing") {
return {
type: "missing",
value: null,
confidence
};
}
return {
type: rawType as NormalizedFragmentV2["time_scope"]["type"],
value: toOptionalString(source.value),
confidence
};
}
const periodType = normalizeToken(source.period_type);
const year = parseYear(source.year);
const month = parseMonth(source.month);
if ((periodType === "year" || (periodType.length === 0 && year !== null)) && year !== null) {
return {
type: "explicit",
value: String(year),
confidence: confidence === "low" ? "medium" : confidence
};
}
if ((periodType === "month" || periodType === "year_month" || (year !== null && month !== null)) && year !== null && month !== null) {
return {
type: "explicit",
value: `${year}-${String(month).padStart(2, "0")}`,
confidence: confidence === "low" ? "medium" : confidence
};
}
}
const inferred = inferTimeScope(rawText);
if (inferred.type !== "missing") {
return inferred;
}
return fallback;
}
function coerceFlags(
value: unknown,
fallback: NormalizedFragmentV2["flags"]
): NormalizedFragmentV2["flags"] {
if (!value || typeof value !== "object") {
return fallback;
}
const source = value as Record<string, unknown>;
const pick = (key: keyof NormalizedFragmentV2["flags"], aliases: string[] = []): boolean => {
if (key in source) {
return coerceBoolean(source[key], fallback[key]);
}
for (const alias of aliases) {
if (alias in source) {
return coerceBoolean(source[alias], fallback[key]);
}
}
return fallback[key];
};
return {
has_multi_entity_scope: pick("has_multi_entity_scope", ["multi_entity_scope"]),
asks_for_chain_explanation: pick("asks_for_chain_explanation", ["asks_for_chain", "chain_explanation"]),
asks_for_ranking_or_top: pick("asks_for_ranking_or_top", ["asks_for_ranking", "asks_for_top"]),
asks_for_period_summary: pick("asks_for_period_summary", ["period_summary"]),
asks_for_rule_check: pick("asks_for_rule_check", ["rule_check"]),
asks_for_anomaly_scan: pick("asks_for_anomaly_scan", ["anomaly_scan"]),
asks_for_exact_object_trace: pick("asks_for_exact_object_trace", ["exact_object_trace"]),
asks_for_evidence: pick("asks_for_evidence", ["evidence"]),
mentions_period_close_context: pick("mentions_period_close_context", ["period_close_context"])
};
}
function mapCandidateLabel(value: string): NormalizedFragmentV2["candidate_labels"][number] | null {
const token = normalizeToken(value);
if (CANDIDATE_LABEL_VALUES.includes(token as NormalizedFragmentV2["candidate_labels"][number])) {
return token as NormalizedFragmentV2["candidate_labels"][number];
}
if (["show_documents", "document_list", "show_docs", "point_answer", "lookup"].includes(token)) {
return "simple_factual";
}
if (["ranking", "top", "summary", "analytical"].includes(token)) {
return "heavy_analytical";
}
if (["chain", "cross", "cross_domain"].includes(token)) {
return "cross_entity";
}
if (["rule_check", "control", "rules"].includes(token)) {
return "rule_based_account_control";
}
if (["risk_scan", "anomaly", "risk"].includes(token)) {
return "anomaly_probe";
}
if (["period_close", "month_close"].includes(token)) {
return "period_close_risk";
}
if (["ambiguous", "unclear"].includes(token)) {
return "ambiguous_human_query";
}
return null;
}
function coerceCandidateLabels(
value: unknown,
flags: NormalizedFragmentV2["flags"],
domainRelevance: NormalizedFragmentV2["domain_relevance"],
fallback: NormalizedFragmentV2["candidate_labels"]
): NormalizedFragmentV2["candidate_labels"] {
const parsed = coerceStringArray(value)
.map((item) => mapCandidateLabel(item))
.filter((item): item is NormalizedFragmentV2["candidate_labels"][number] => Boolean(item));
if (parsed.length > 0) {
return Array.from(new Set(parsed));
}
const inferred = pickCandidateLabels(flags, domainRelevance);
if (inferred.length > 0) {
return inferred;
}
return fallback;
}
function coerceFragmentV2(rawFragment: unknown, index: number, userMessage: string): NormalizedFragmentV2 | null {
const source = rawFragment && typeof rawFragment === "object" ? (rawFragment as Record<string, unknown>) : {};
const rawText =
toOptionalString(source.raw_fragment_text) ??
toOptionalString(source.rawText) ??
toOptionalString(source.fragment_text) ??
toOptionalString(source.text) ??
userMessage.trim();
const base = buildFragmentV2(rawText, index) ?? buildFragmentV2(userMessage, index);
if (!base) {
return null;
}
const domainRelevance = coerceDomainRelevance(source.domain_relevance, base.domain_relevance);
const businessScope = coerceBusinessScope(source.business_scope, base.business_scope, domainRelevance);
const flags = coerceFlags(source.flags, base.flags);
const entityHints = coerceStringArray(source.entity_hints);
const accountHints = coerceStringArray(source.account_hints);
const documentHints = coerceStringArray(source.document_hints);
const registerHints = coerceStringArray(source.register_hints);
return {
fragment_id: coerceFragmentId(source.fragment_id, index, base.fragment_id),
raw_fragment_text: rawText,
normalized_fragment_text: toOptionalString(source.normalized_fragment_text) ?? base.normalized_fragment_text,
domain_relevance: domainRelevance,
business_scope: businessScope,
entity_hints: entityHints.length > 0 ? entityHints : base.entity_hints,
account_hints: accountHints.length > 0 ? accountHints : base.account_hints,
document_hints: documentHints.length > 0 ? documentHints : base.document_hints,
register_hints: registerHints.length > 0 ? registerHints : base.register_hints,
time_scope: coerceTimeScope(source.time_scope, rawText, base.time_scope),
flags,
candidate_labels: coerceCandidateLabels(source.candidate_labels, flags, domainRelevance, base.candidate_labels),
confidence: coerceConfidence(source.confidence, base.confidence)
};
}
function coerceDiscardedFragments(value: unknown): DiscardedFragmentV2[] {
if (!Array.isArray(value)) {
return [];
}
const collected: DiscardedFragmentV2[] = [];
for (const item of value) {
if (!item || typeof item !== "object") {
continue;
}
const source = item as Record<string, unknown>;
const raw = toOptionalString(source.raw_fragment_text);
const reason = toOptionalString(source.reason);
if (!raw || !reason) {
continue;
}
collected.push({
raw_fragment_text: raw,
reason
});
}
return collected;
}
function coerceScopeConfidence(
value: unknown,
fallback: NormalizedQueryV2["scope_confidence"]
): NormalizedQueryV2["scope_confidence"] {
return coerceConfidence(value, fallback);
}
function coerceGlobalNotes(
value: unknown,
fallbackNeedsClarification: boolean
): NormalizedQueryV2["global_notes"] {
if (!value || typeof value !== "object") {
return {
needs_clarification: fallbackNeedsClarification,
clarification_reason: fallbackNeedsClarification ? "clarification_required" : null
};
}
const source = value as Record<string, unknown>;
const needs = coerceBoolean(source.needs_clarification, fallbackNeedsClarification);
const clarificationReason = toOptionalString(source.clarification_reason);
return {
needs_clarification: needs,
clarification_reason: needs ? clarificationReason ?? "clarification_required" : null
};
}
function coerceNormalizedCandidateV2(candidate: unknown, userMessage: string): NormalizedQueryV2 | null {
if (!candidate || typeof candidate !== "object") {
return null;
}
const source = candidate as Record<string, unknown>;
const sourceFragments = Array.isArray(source.fragments)
? source.fragments
: source.fragment && typeof source.fragment === "object"
? [source.fragment]
: splitIntoCandidateFragments(userMessage).map((text) => ({ raw_fragment_text: text }));
const fragments = sourceFragments
.map((item, index) => coerceFragmentV2(item, index, userMessage))
.filter((item): item is NormalizedFragmentV2 => item !== null);
const inScopeCount = fragments.filter((item) => item.domain_relevance === "in_scope").length;
const unclearCount = fragments.filter((item) => item.domain_relevance === "unclear").length;
const messageInScope = inScopeCount > 0;
const inferredScopeConfidence: NormalizedQueryV2["scope_confidence"] = messageInScope ? (unclearCount > 0 ? "medium" : "high") : "low";
const inferredNeedsClarification = messageInScope && (unclearCount > 0 || fragments.some((item) => item.time_scope.type === "missing"));
return {
schema_version: "normalized_query_v2",
user_message_raw: toOptionalString(source.user_message_raw) ?? userMessage,
message_in_scope: coerceBoolean(source.message_in_scope, messageInScope),
scope_confidence: coerceScopeConfidence(source.scope_confidence, inferredScopeConfidence),
contains_multiple_tasks: coerceBoolean(source.contains_multiple_tasks, fragments.length > 1),
fragments,
discarded_fragments: coerceDiscardedFragments(source.discarded_fragments),
global_notes: coerceGlobalNotes(source.global_notes, inferredNeedsClarification)
};
}
function collectDateSpans(text: string): Array<{ start: number; end: number }> {
const spans: Array<{ start: number; end: number }> = [];
const patterns = [
@@ -1099,6 +1541,7 @@ export class NormalizerService {
try {
normalizedCandidate = safeJsonParse(outputText);
if (schemaVersion !== "v1") {
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
}
if (schemaVersion === "v2_0_2") {
@@ -1150,6 +1593,7 @@ export class NormalizerService {
try {
normalizedCandidate = safeJsonParse(outputText);
if (schemaVersion !== "v1") {
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
}
if (schemaVersion === "v2_0_2") {