АДРЕСНЫЙ РЕЖИМ - ллм декомпоз
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import type { AddressFilterExtraction, AddressFilterSet, AddressIntent } from "../types/addressQuery";
|
||||
import iconv from "iconv-lite";
|
||||
|
||||
const ACCOUNT_PATTERN = /(?:сч[её]т|счет|account)[^0-9]{0,12}(\d{2}(?:[.,]\d{1,2})?)/i;
|
||||
const LIMIT_PATTERN = /(?:\btop\b|\blimit\b|\bпервые\b|\bтоп\b)\s*(\d{1,3})/i;
|
||||
@@ -14,8 +15,10 @@ const YEAR_RANGE_PATTERN =
|
||||
/(?:за|for|с|from)?\s*(20\d{2})\s*(?:[-‐‑‒–—―−]|до|to|по)\s*(20\d{2})(?:\s*(?:г(?:од|ода)?\.?|year))?(?=[^\d]|$)/iu;
|
||||
const YEAR_RANGE_LOOSE_PATTERN = /\b(20\d{2})\b\s*(?:[-‐‑‒–—―−]|до|to|по)\s*\b(20\d{2})\b/iu;
|
||||
const YEAR_PERIOD_PATTERN =
|
||||
/(?:за|for)\s*(20\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year)?/iu;
|
||||
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year)(?=$|[\s,.;:!?()\-])/iu;
|
||||
/(?:за|for)\s*(20\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*20\d{2})\s*(?:г(?:од|ода)?\.?|year|god)?/iu;
|
||||
const YEAR_PERIOD_SHORT_PATTERN = /(?:^|[\s,.;:!?()\-])(\d{2})\s*(?:г(?:од|ода)?\.?|year|god)(?=$|[\s,.;:!?()\-])/iu;
|
||||
const YEAR_PERIOD_SHORT_ORDINAL_PATTERN =
|
||||
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(\d{2})\s*(?:[-\s]?(?:й|ый|ой|th))(?:\s*(?:г(?:од|ода)?\.?|year|period|период))?(?=$|[\s,.;:!?()\-])/iu;
|
||||
const YEAR_PERIOD_ANY_PATTERN =
|
||||
/(?:^|[\s,.;:!?()\-])((?:19|20)\d{2})(?!\s*(?:[-‐‑‒–—―−]|до|to|по)\s*(?:19|20)\d{2})(?![.\/-]\d)(?:\s*(?:г(?:од|ода)?\.?|year))?(?=$|[\s,.;:!?()\-])/iu;
|
||||
const MONTH_PERIOD_NUMERIC_MONTH_YEAR_PATTERN =
|
||||
@@ -26,6 +29,58 @@ const MONTH_PERIOD_NAME_PATTERN =
|
||||
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*([a-zа-яё]+)\s+(20\d{2})(?:\s*г(?:од|ода|\\.)?)?(?=$|[\s,.;:!?()\-])/iu;
|
||||
const MONTH_PERIOD_NAME_YEAR_FIRST_PATTERN =
|
||||
/(?:^|[\s,.;:!?()\-])(?:за|for|на|in)?\s*(20\d{2})(?:\s*г(?:од|ода|\\.)?)?\s+([a-zа-яё]+)(?=$|[\s,.;:!?()\-])/iu;
|
||||
const DOC_SIGNAL_PATTERN =
|
||||
"(?:док(?:и|умент|ументы|ументов|умам|ума)|docs?|documents?|doki|dokument(?:y|ov|am|a)?)";
|
||||
|
||||
function textMojibakeScore(value: string): number {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
|
||||
}
|
||||
|
||||
function looksLikeMojibake(value: string): boolean {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
|
||||
}
|
||||
|
||||
function decodeUtf8FromWin1251Mojibake(value: string): string {
|
||||
if (!looksLikeMojibake(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
const bytes = iconv.encode(value, "win1251");
|
||||
const decoded = bytes.toString("utf8");
|
||||
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeUtf8FromLatin1Mojibake(value: string): string {
|
||||
if (!looksLikeMojibake(value)) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
const decoded = Buffer.from(value, "latin1").toString("utf8");
|
||||
return textMojibakeScore(decoded) > textMojibakeScore(value) ? decoded : value;
|
||||
} catch {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeMojibakeString(value: string): string {
|
||||
const fromWin1251 = decodeUtf8FromWin1251Mojibake(value);
|
||||
return decodeUtf8FromLatin1Mojibake(fromWin1251);
|
||||
}
|
||||
|
||||
function toIsoDate(year: number, month: number, day: number): string | null {
|
||||
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
||||
@@ -200,18 +255,30 @@ function extractYearPeriod(text: string): { period_from?: string; period_to?: st
|
||||
}
|
||||
|
||||
const shortYearMatch = text.match(YEAR_PERIOD_SHORT_PATTERN);
|
||||
if (!shortYearMatch) {
|
||||
return {};
|
||||
if (shortYearMatch) {
|
||||
const shortYear = Number(shortYearMatch[1]);
|
||||
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
|
||||
const year = 2000 + shortYear;
|
||||
return {
|
||||
period_from: `${year}-01-01`,
|
||||
period_to: `${year}-12-31`
|
||||
};
|
||||
}
|
||||
}
|
||||
const shortYear = Number(shortYearMatch[1]);
|
||||
if (!Number.isFinite(shortYear) || shortYear < 0 || shortYear > 99) {
|
||||
return {};
|
||||
|
||||
const shortOrdinalMatch = text.match(YEAR_PERIOD_SHORT_ORDINAL_PATTERN);
|
||||
if (shortOrdinalMatch) {
|
||||
const shortYear = Number(shortOrdinalMatch[1]);
|
||||
if (Number.isFinite(shortYear) && shortYear >= 0 && shortYear <= 99) {
|
||||
const year = 2000 + shortYear;
|
||||
return {
|
||||
period_from: `${year}-01-01`,
|
||||
period_to: `${year}-12-31`
|
||||
};
|
||||
}
|
||||
}
|
||||
const year = 2000 + shortYear;
|
||||
return {
|
||||
period_from: `${year}-01-01`,
|
||||
period_to: `${year}-12-31`
|
||||
};
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
function extractYearRangePeriod(text: string): { period_from?: string; period_to?: string } {
|
||||
@@ -374,14 +441,36 @@ function isLikelyCounterpartyToken(rawToken: string): boolean {
|
||||
"нахуй",
|
||||
"покеж",
|
||||
"покажи",
|
||||
"выведи"
|
||||
"показать",
|
||||
"выведи",
|
||||
"show",
|
||||
"list",
|
||||
"please",
|
||||
"vse",
|
||||
"all",
|
||||
"kakie",
|
||||
"kakoi",
|
||||
"est",
|
||||
"pokaji",
|
||||
"pokazhi",
|
||||
"pokazh",
|
||||
"pokezh",
|
||||
"doki",
|
||||
"doky",
|
||||
"dokument",
|
||||
"dokumenty",
|
||||
"documents",
|
||||
"docs"
|
||||
]);
|
||||
return !stopWords.has(lowered);
|
||||
}
|
||||
|
||||
function hasDocsOrBankSignal(text: string): boolean {
|
||||
const lowered = String(text ?? "").toLowerCase();
|
||||
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|transactions?|bank\s+ops|bank\s+operations?)/iu.test(
|
||||
return new RegExp(
|
||||
`(?:${DOC_SIGNAL_PATTERN}|банк|выписк|платеж|платёж|оплат|transactions?|bank\\s+ops|bank\\s+operations?|payment|payments?|platezh|oplata)`,
|
||||
"iu"
|
||||
).test(
|
||||
lowered
|
||||
);
|
||||
}
|
||||
@@ -448,9 +537,11 @@ function extractCounterpartyFromFreeTextHeuristic(text: string): string | undefi
|
||||
|
||||
function extractImplicitCounterpartyValue(text: string): string | undefined {
|
||||
const input = String(text ?? "");
|
||||
const beforeDocsMatch = input.match(
|
||||
/(?:^|\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\s+(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)(?=[\s,.;:!?)]|$)/iu
|
||||
const beforeDocsPattern = new RegExp(
|
||||
`(?:^|\\s)([a-zа-яё][a-zа-яё0-9._-]{1,})\\s+${DOC_SIGNAL_PATTERN}(?=[\\s,.;:!?)]|$)`,
|
||||
"iu"
|
||||
);
|
||||
const beforeDocsMatch = input.match(beforeDocsPattern);
|
||||
if (beforeDocsMatch) {
|
||||
const candidate = String(beforeDocsMatch[1] ?? "").trim();
|
||||
if (isLikelyCounterpartyToken(candidate)) {
|
||||
@@ -458,9 +549,11 @@ function extractImplicitCounterpartyValue(text: string): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
const afterDocsMatch = input.match(
|
||||
/(?:док(?:и|ум(?:ент(?:ы|ов|ам|а)?)?)|docs?|documents?)\s+(?:по\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\s,.;:!?)]|$)/iu
|
||||
const afterDocsPattern = new RegExp(
|
||||
`${DOC_SIGNAL_PATTERN}\\s+(?:по\\s+)?([a-zа-яё][a-zа-яё0-9._-]{1,})(?=[\\s,.;:!?)]|$)`,
|
||||
"iu"
|
||||
);
|
||||
const afterDocsMatch = input.match(afterDocsPattern);
|
||||
if (afterDocsMatch) {
|
||||
const candidate = String(afterDocsMatch[1] ?? "").trim();
|
||||
if (isLikelyCounterpartyToken(candidate)) {
|
||||
@@ -488,7 +581,8 @@ function requiredFiltersByIntent(intent: AddressIntent): Array<keyof AddressFilt
|
||||
}
|
||||
|
||||
export function extractAddressFilters(userMessage: string, intent: AddressIntent): AddressFilterExtraction {
|
||||
const text = String(userMessage ?? "").trim();
|
||||
const rawText = String(userMessage ?? "").trim();
|
||||
const text = normalizeMojibakeString(rawText);
|
||||
const filters: AddressFilterSet = {
|
||||
sort: "period_desc",
|
||||
limit: 20
|
||||
@@ -628,3 +722,5 @@ export function extractAddressFilters(userMessage: string, intent: AddressIntent
|
||||
warnings
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -11,12 +11,17 @@ const ADDRESS_ACTION_TOKENS = [
|
||||
"debt",
|
||||
"owe",
|
||||
"покажи",
|
||||
"покаж",
|
||||
"показ",
|
||||
"список",
|
||||
"найди",
|
||||
"найд",
|
||||
"выведи",
|
||||
"вывед",
|
||||
"кто",
|
||||
"кому",
|
||||
"какие",
|
||||
"что по",
|
||||
"остаток",
|
||||
"долг",
|
||||
"задолж",
|
||||
@@ -70,6 +75,10 @@ const ADDRESS_ENTITY_TOKENS = [
|
||||
"кредитор",
|
||||
"аванс",
|
||||
"оплат",
|
||||
"поступлен",
|
||||
"поступлени",
|
||||
"списан",
|
||||
"списани",
|
||||
"долг",
|
||||
"должен",
|
||||
"должны",
|
||||
@@ -142,6 +151,83 @@ function hasAddressFollowupSignal(text: string): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
function hasDocsOrBankSignal(text: string): boolean {
|
||||
return /(?:док(?:и|умент|ументы|ументов)|docs?|documents?|банк|выписк|платеж|платёж|оплат|поступлен|списан|transactions?|bank\s+ops|bank\s+operations?)/iu.test(
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
function hasLikelyCounterpartyToken(text: string): boolean {
|
||||
const stopWords = new Set([
|
||||
"за",
|
||||
"с",
|
||||
"по",
|
||||
"на",
|
||||
"и",
|
||||
"или",
|
||||
"док",
|
||||
"доки",
|
||||
"документ",
|
||||
"документы",
|
||||
"документов",
|
||||
"банк",
|
||||
"банковские",
|
||||
"операции",
|
||||
"платежи",
|
||||
"платеж",
|
||||
"платёж",
|
||||
"контрагент",
|
||||
"контрагенту",
|
||||
"контрагента",
|
||||
"компания",
|
||||
"компании",
|
||||
"организация",
|
||||
"организации",
|
||||
"год",
|
||||
"года",
|
||||
"г",
|
||||
"плс",
|
||||
"pls",
|
||||
"пж",
|
||||
"пжлст",
|
||||
"пожалуйста",
|
||||
"бля",
|
||||
"блять",
|
||||
"епт",
|
||||
"ёпт",
|
||||
"епта",
|
||||
"нах",
|
||||
"нахуй",
|
||||
"покеж",
|
||||
"покажи",
|
||||
"показать",
|
||||
"покаж",
|
||||
"выведи",
|
||||
"show",
|
||||
"list",
|
||||
"please",
|
||||
"all",
|
||||
"vse"
|
||||
]);
|
||||
const tokens = String(text ?? "")
|
||||
.split(/[^a-zа-яё0-9._-]+/iu)
|
||||
.map((token) => token.trim())
|
||||
.filter((token) => token.length >= 2);
|
||||
return tokens.some((token) => {
|
||||
const lowered = token.toLowerCase();
|
||||
if (stopWords.has(lowered)) {
|
||||
return false;
|
||||
}
|
||||
if (/^\d+$/.test(lowered)) {
|
||||
return false;
|
||||
}
|
||||
if (/^(?:19|20)\d{2}$/.test(lowered)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function hasAnyToken(text: string, tokens: string[]): boolean {
|
||||
return tokens.some((token) => text.includes(token));
|
||||
}
|
||||
@@ -186,6 +272,14 @@ export function detectAddressQuestionMode(userMessage: string): AddressModeDetec
|
||||
};
|
||||
}
|
||||
|
||||
if (!hasDeepReasoning && hasDocsOrBankSignal(text) && (hasLooseByAnchor || hasLikelyCounterpartyToken(text))) {
|
||||
return {
|
||||
mode: "address_query",
|
||||
confidence: "medium",
|
||||
reasons: ["docs_or_bank_signal_detected", "anchor_like_token_detected"]
|
||||
};
|
||||
}
|
||||
|
||||
if (hasDeepReasoning) {
|
||||
return {
|
||||
mode: "deep_analysis",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type {
|
||||
import type {
|
||||
AddressFilterSet,
|
||||
AddressIntent,
|
||||
AddressIntentResolution,
|
||||
@@ -45,7 +45,14 @@ function toNonEmptyString(value: unknown): string | null {
|
||||
}
|
||||
|
||||
function hasAllTimeHint(text: string): boolean {
|
||||
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(
|
||||
const normalized = String(text ?? "");
|
||||
return /(?:за\s+вс[её]\s+время|за\s+весь\s+период|за\s+весь\s+срок|за\s+всю\s+истори(?:ю|и)|за\s+любой\s+период|за\s+любой\s+срок|for\s+all\s+time|all\s+time|for\s+entire\s+period|entire\s+period|for\s+any\s+period|any\s+period|for\s+full\s+history|full\s+history)/iu.test(
|
||||
normalized
|
||||
);
|
||||
}
|
||||
|
||||
function hasSameDateHint(text: string): boolean {
|
||||
return /(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|та\s+же\s+дата|same\s+date|as\s+of\s+same\s+date|the\s+same\s+date)/iu.test(
|
||||
String(text ?? "")
|
||||
);
|
||||
}
|
||||
@@ -58,10 +65,23 @@ export function hasAddressFollowupContextSignal(text: string): boolean {
|
||||
if (hasAllTimeHint(normalized)) {
|
||||
return true;
|
||||
}
|
||||
if (/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that)/iu.test(normalized)) {
|
||||
if (
|
||||
/(?:^|\s)(?:и|а\s+еще|а\s+ещё|еще|ещё|также|а\s+теперь|теперь|по\s+этому|по\s+тому|это\s+же|в\s+этом|тот\s+же|also|same|that|then|now)/iu.test(
|
||||
normalized
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return normalized.split(/\s+/).filter(Boolean).length <= 8;
|
||||
if (hasSameDateHint(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const tokenCount = normalized.split(/\s+/).filter(Boolean).length;
|
||||
const hasPeriodLiteral = /\b(?:19|20)\d{2}(?:[./-](?:0?[1-9]|1[0-2]))?\b/.test(normalized);
|
||||
if (tokenCount <= 8 && hasPeriodLiteral) {
|
||||
return true;
|
||||
}
|
||||
return tokenCount <= 6;
|
||||
}
|
||||
|
||||
function mergeFollowupFilters(
|
||||
@@ -81,7 +101,11 @@ function mergeFollowupFilters(
|
||||
const previousCounterparty = toNonEmptyString(previous.counterparty);
|
||||
const previousContract = toNonEmptyString(previous.contract);
|
||||
const previousAccount = toNonEmptyString(previous.account);
|
||||
const previousAsOfDate = toNonEmptyString(previous.as_of_date);
|
||||
const previousPeriodFrom = toNonEmptyString(previous.period_from);
|
||||
const previousPeriodTo = toNonEmptyString(previous.period_to);
|
||||
const allTimeRequested = hasAllTimeHint(userMessage);
|
||||
const sameDateRequested = hasSameDateHint(userMessage);
|
||||
|
||||
if (intent === "list_documents_by_counterparty" || intent === "bank_operations_by_counterparty") {
|
||||
if (!toNonEmptyString(merged.counterparty)) {
|
||||
@@ -97,21 +121,24 @@ function mergeFollowupFilters(
|
||||
|
||||
if (intent === "account_balance_snapshot" || intent === "documents_forming_balance") {
|
||||
if (!toNonEmptyString(merged.account)) {
|
||||
const inheritedAccount =
|
||||
previousAccount ??
|
||||
(followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
|
||||
const inheritedAccount = previousAccount ?? (followupContext.previous_anchor_type === "account" ? previousAnchorValue : null);
|
||||
if (inheritedAccount) {
|
||||
merged.account = inheritedAccount;
|
||||
reasons.push("account_from_followup_context");
|
||||
}
|
||||
}
|
||||
if (sameDateRequested) {
|
||||
const inheritedAsOfDate = previousAsOfDate ?? previousPeriodTo ?? previousPeriodFrom;
|
||||
if (inheritedAsOfDate && merged.as_of_date !== inheritedAsOfDate) {
|
||||
merged.as_of_date = inheritedAsOfDate;
|
||||
reasons.push("as_of_date_from_followup_context");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (intent === "open_items_by_counterparty_or_contract" || intent === "list_open_contracts") {
|
||||
if (!toNonEmptyString(merged.contract)) {
|
||||
const inheritedContract =
|
||||
previousContract ??
|
||||
(followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
|
||||
const inheritedContract = previousContract ?? (followupContext.previous_anchor_type === "contract" ? previousAnchorValue : null);
|
||||
if (inheritedContract) {
|
||||
merged.contract = inheritedContract;
|
||||
reasons.push("contract_from_followup_context");
|
||||
@@ -140,11 +167,11 @@ function mergeFollowupFilters(
|
||||
const currentHasPeriod = hasExplicitPeriodWindow(merged);
|
||||
const previousHasPeriod = hasExplicitPeriodWindow(previous);
|
||||
if (!currentHasPeriod && previousHasPeriod && hasAddressFollowupContextSignal(userMessage)) {
|
||||
if (toNonEmptyString(previous.period_from)) {
|
||||
merged.period_from = previous.period_from;
|
||||
if (previousPeriodFrom) {
|
||||
merged.period_from = previousPeriodFrom;
|
||||
}
|
||||
if (toNonEmptyString(previous.period_to)) {
|
||||
merged.period_to = previous.period_to;
|
||||
if (previousPeriodTo) {
|
||||
merged.period_to = previousPeriodTo;
|
||||
}
|
||||
reasons.push("period_from_followup_context");
|
||||
}
|
||||
@@ -241,4 +268,3 @@ export function runAddressDecomposeStage(
|
||||
baseReasons
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import * as companyAnchorResolver_1 from "./companyAnchorResolver";
|
||||
import * as assistantRuntimeGuards_1 from "./assistantRuntimeGuards";
|
||||
import * as assistantClaimBoundEvidence_1 from "./assistantClaimBoundEvidence";
|
||||
import * as addressQueryService_1 from "./addressQueryService";
|
||||
import iconv from "iconv-lite";
|
||||
function retrievalSummaryForRoute(route) {
|
||||
if (route === "store_canonical")
|
||||
return "Canonical accounting data path selected.";
|
||||
@@ -1386,7 +1387,7 @@ function hasAccountingSignal(text) {
|
||||
}
|
||||
function hasFollowupMarker(text) {
|
||||
const compact = compactWhitespace(text.toLowerCase());
|
||||
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|plus|also|dobav|utochn|prodolzh)/i.test(compact);
|
||||
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|а теперь|теперь|plus|also|dobav|utochn|prodolzh|then|now)/i.test(compact);
|
||||
}
|
||||
function hasReferentialPointer(text) {
|
||||
return /(по этому|по тому|это же|этой|этим|этому|из этого|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
|
||||
@@ -1780,6 +1781,153 @@ function toNonEmptyString(value) {
|
||||
const text = String(value).trim();
|
||||
return text.length > 0 ? text : null;
|
||||
}
|
||||
const ADDRESS_PREDECOMPOSE_NOISE_TOKENS = new Set([
|
||||
"за",
|
||||
"с",
|
||||
"по",
|
||||
"на",
|
||||
"и",
|
||||
"или",
|
||||
"док",
|
||||
"доки",
|
||||
"docs",
|
||||
"documents",
|
||||
"doki",
|
||||
"dokument",
|
||||
"dokumenty",
|
||||
"документ",
|
||||
"документы",
|
||||
"документов",
|
||||
"банк",
|
||||
"банковские",
|
||||
"операции",
|
||||
"платеж",
|
||||
"платёж",
|
||||
"платежи",
|
||||
"контрагент",
|
||||
"контрагенту",
|
||||
"контрагента",
|
||||
"год",
|
||||
"года",
|
||||
"г",
|
||||
"year",
|
||||
"god",
|
||||
"плс",
|
||||
"pls",
|
||||
"пж",
|
||||
"пжлст",
|
||||
"пожалуйста",
|
||||
"please",
|
||||
"покеж",
|
||||
"покажи",
|
||||
"показать",
|
||||
"show",
|
||||
"list",
|
||||
"выведи",
|
||||
"которые",
|
||||
"какие",
|
||||
"какой",
|
||||
"есть",
|
||||
"est",
|
||||
"kakie",
|
||||
"kakoi",
|
||||
"vse",
|
||||
"all",
|
||||
"blya",
|
||||
"blyat",
|
||||
"епт",
|
||||
"ёпт",
|
||||
"бля"
|
||||
]);
|
||||
function textMojibakeScoreForAddress(value) {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
|
||||
}
|
||||
function looksLikeMojibakeForAddress(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
|
||||
}
|
||||
function repairAddressMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibakeForAddress(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
try {
|
||||
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
}
|
||||
}
|
||||
catch (_error) { }
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScoreForAddress(fromLatin1) > textMojibakeScoreForAddress(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
}
|
||||
}
|
||||
catch (_error) { }
|
||||
return candidate;
|
||||
}
|
||||
function extractAddressAnchorTokens(value) {
|
||||
const source = repairAddressMojibake(compactWhitespace(String(value ?? "").toLowerCase()));
|
||||
if (!source) {
|
||||
return [];
|
||||
}
|
||||
const tokens = source
|
||||
.split(/[^a-zа-яё0-9._-]+/iu)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length >= 2);
|
||||
const filtered = [];
|
||||
for (const token of tokens) {
|
||||
if (/^\d+$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:19|20)\d{2}$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:0?[1-9]|1[0-2])[./-](?:19|20)\d{2}$/.test(token) || /^(?:19|20)\d{2}[./-](?:0?[1-9]|1[0-2])$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:янв|фев|мар|апр|май|июн|июл|авг|сен|сент|окт|ноя|дек|january|february|march|april|may|june|july|august|september|october|november|december)/i.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (ADDRESS_PREDECOMPOSE_NOISE_TOKENS.has(token)) {
|
||||
continue;
|
||||
}
|
||||
filtered.push(token);
|
||||
}
|
||||
return Array.from(new Set(filtered));
|
||||
}
|
||||
function selectPreferredAddressFragmentCandidate(rawText, normalizedText) {
|
||||
const normalizedCandidate = compactWhitespace(repairAddressMojibake(normalizedText ?? ""));
|
||||
const rawCandidate = compactWhitespace(repairAddressMojibake(rawText ?? ""));
|
||||
if (!normalizedCandidate && !rawCandidate) {
|
||||
return null;
|
||||
}
|
||||
if (!normalizedCandidate) {
|
||||
return rawCandidate;
|
||||
}
|
||||
if (!rawCandidate) {
|
||||
return normalizedCandidate;
|
||||
}
|
||||
const normalizedAnchors = extractAddressAnchorTokens(normalizedCandidate);
|
||||
const rawAnchors = extractAddressAnchorTokens(rawCandidate);
|
||||
if (rawAnchors.length > 0 && normalizedAnchors.length === 0) {
|
||||
return rawCandidate;
|
||||
}
|
||||
return normalizedCandidate;
|
||||
}
|
||||
function readAddressFilterString(addressDebug, key) {
|
||||
const filters = addressDebug?.extracted_filters;
|
||||
if (!filters || typeof filters !== "object") {
|
||||
@@ -1801,7 +1949,8 @@ function findLastAddressAssistantDebug(items) {
|
||||
return null;
|
||||
}
|
||||
function hasAddressFollowupContextSignal(userMessage) {
|
||||
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
|
||||
const repaired = repairAddressMojibake(String(userMessage ?? ""));
|
||||
const text = compactWhitespace(repaired.toLowerCase());
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
@@ -1811,10 +1960,16 @@ function hasAddressFollowupContextSignal(userMessage) {
|
||||
if (hasReferentialPointer(text)) {
|
||||
return true;
|
||||
}
|
||||
if (/(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|same\s+date|the\s+same\s+date|as\s+of\s+same\s+date)/iu.test(text)) {
|
||||
return true;
|
||||
}
|
||||
const shortFollowup = countTokens(text) <= 8;
|
||||
if (shortFollowup && hasFollowupMarker(text)) {
|
||||
return true;
|
||||
}
|
||||
if (shortFollowup && hasPeriodLiteral(text)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function resolveAddressFollowupCarryoverContext(userMessage, items) {
|
||||
@@ -1851,11 +2006,12 @@ function resolveAddressFollowupCarryoverContext(userMessage, items) {
|
||||
};
|
||||
}
|
||||
function isAddressLlmPreDecomposeCandidate(userMessage) {
|
||||
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
|
||||
const repaired = repairAddressMojibake(String(userMessage ?? ""));
|
||||
const text = compactWhitespace(repaired.toLowerCase());
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|банк|выписк|платеж|оплат|поступлен|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?)/i.test(text);
|
||||
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|сальдо|банк|выписк|платеж|оплат|поступлен|поступлени|списан|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?|doki|dokument(?:y|ov|am|a)?|platezh|oplata|schet|saldo)/i.test(text);
|
||||
}
|
||||
function extractAddressQuestionFromNormalized(normalized) {
|
||||
if (!normalized || typeof normalized !== "object") {
|
||||
@@ -1873,13 +2029,16 @@ function extractAddressQuestionFromNormalized(normalized) {
|
||||
continue;
|
||||
}
|
||||
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
|
||||
if (readiness === "no_route") {
|
||||
continue;
|
||||
}
|
||||
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
|
||||
const rawText = toNonEmptyString(fragment.raw_fragment_text);
|
||||
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
|
||||
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
|
||||
if (!candidate) {
|
||||
continue;
|
||||
}
|
||||
if (candidate.length >= 3 && candidate.length <= 500) {
|
||||
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
|
||||
continue;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -1986,13 +2145,16 @@ function extractAddressQuestionFromRawNormalizerOutput(rawModelOutput) {
|
||||
continue;
|
||||
}
|
||||
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
|
||||
if (readiness === "no_route") {
|
||||
continue;
|
||||
}
|
||||
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
|
||||
const rawText = toNonEmptyString(fragment.raw_fragment_text);
|
||||
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
|
||||
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
|
||||
if (!candidate) {
|
||||
continue;
|
||||
}
|
||||
if (candidate.length >= 3 && candidate.length <= 500) {
|
||||
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
|
||||
continue;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -2203,12 +2365,21 @@ export class AssistantService {
|
||||
reason: "disabled_by_feature_flag"
|
||||
};
|
||||
const addressInputMessage = toNonEmptyString(addressPreDecompose?.effectiveMessage) ?? userMessage;
|
||||
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
|
||||
const shouldPreferContextualLane = Boolean(carryover?.followupContext);
|
||||
if (shouldPreferContextualLane) {
|
||||
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
|
||||
followupContext: carryover.followupContext
|
||||
});
|
||||
if (contextualAddressLane?.handled) {
|
||||
return finalizeAddressLaneResponse(contextualAddressLane, addressInputMessage, carryover, addressPreDecompose);
|
||||
}
|
||||
}
|
||||
const primaryAddressLane = await this.addressQueryService.tryHandle(addressInputMessage);
|
||||
if (primaryAddressLane?.handled) {
|
||||
return finalizeAddressLaneResponse(primaryAddressLane, addressInputMessage, null, addressPreDecompose);
|
||||
}
|
||||
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
|
||||
if (carryover?.followupContext) {
|
||||
if (!shouldPreferContextualLane && carryover?.followupContext) {
|
||||
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
|
||||
followupContext: carryover.followupContext
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@ import { toRouteHintSummary } from "./routeHintAdapter";
|
||||
import { validateNormalized } from "./schemaValidator";
|
||||
import { redactRequestPayload, saveEvalCase, saveTrace, type TraceRecord } from "./traceLogger";
|
||||
import type {
|
||||
DiscardedFragmentV2,
|
||||
ExecutionReadiness,
|
||||
NoRouteReason,
|
||||
NormalizeRequestPayload,
|
||||
@@ -27,6 +28,24 @@ const RETRY_INSTRUCTION_V1 = "IMPORTANT: return valid JSON strictly matching sch
|
||||
const RETRY_INSTRUCTION_V2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2. No markdown.";
|
||||
const RETRY_INSTRUCTION_V2_0_1 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_1. No markdown.";
|
||||
const RETRY_INSTRUCTION_V2_0_2 = "IMPORTANT: return valid JSON strictly matching schema normalized_query_v2_0_2. No markdown.";
|
||||
const CONFIDENCE_LEVELS: ReadonlyArray<NormalizedFragmentV2["confidence"]> = ["high", "medium", "low"];
|
||||
const DOMAIN_RELEVANCE_VALUES: ReadonlyArray<NormalizedFragmentV2["domain_relevance"]> = ["in_scope", "out_of_scope", "unclear"];
|
||||
const BUSINESS_SCOPE_VALUES: ReadonlyArray<NormalizedFragmentV2["business_scope"]> = [
|
||||
"company_specific_accounting",
|
||||
"generic_accounting",
|
||||
"offtopic",
|
||||
"unclear"
|
||||
];
|
||||
const CANDIDATE_LABEL_VALUES: ReadonlyArray<NormalizedFragmentV2["candidate_labels"][number]> = [
|
||||
"heavy_analytical",
|
||||
"cross_entity",
|
||||
"drilldown_explain",
|
||||
"rule_based_account_control",
|
||||
"anomaly_probe",
|
||||
"period_close_risk",
|
||||
"ambiguous_human_query",
|
||||
"simple_factual"
|
||||
];
|
||||
|
||||
function safeJsonParse(text: string): unknown {
|
||||
const cleaned = text.trim().replace(/^```json\s*/i, "").replace(/^```\s*/i, "").replace(/```$/i, "").trim();
|
||||
@@ -79,6 +98,429 @@ function computeRetryMaxOutputTokens(current: number, rawModelResponse: unknown)
|
||||
return Math.min(escalated, 2400);
|
||||
}
|
||||
|
||||
function normalizeToken(value: unknown): string {
|
||||
return String(value ?? "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[\s-]+/g, "_");
|
||||
}
|
||||
|
||||
function toOptionalString(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function coerceBoolean(value: unknown, fallback = false): boolean {
|
||||
if (typeof value === "boolean") {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "number") {
|
||||
if (value === 1) return true;
|
||||
if (value === 0) return false;
|
||||
return fallback;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const token = value.trim().toLowerCase();
|
||||
if (["true", "1", "yes", "y", "да", "ok"].includes(token)) {
|
||||
return true;
|
||||
}
|
||||
if (["false", "0", "no", "n", "нет"].includes(token)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceStringArray(value: unknown): string[] {
|
||||
if (Array.isArray(value)) {
|
||||
return Array.from(
|
||||
new Set(
|
||||
value
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0)
|
||||
)
|
||||
);
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
return Array.from(
|
||||
new Set(
|
||||
value
|
||||
.split(/[,\n;]+/)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0)
|
||||
)
|
||||
);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function coerceConfidence(value: unknown, fallback: NormalizedFragmentV2["confidence"]): NormalizedFragmentV2["confidence"] {
|
||||
if (typeof value === "string") {
|
||||
const token = normalizeToken(value);
|
||||
if (CONFIDENCE_LEVELS.includes(token as NormalizedFragmentV2["confidence"])) {
|
||||
return token as NormalizedFragmentV2["confidence"];
|
||||
}
|
||||
}
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const normalized = value > 1 ? value / 100 : value;
|
||||
if (normalized >= 0.75) return "high";
|
||||
if (normalized >= 0.45) return "medium";
|
||||
return "low";
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceDomainRelevance(
|
||||
value: unknown,
|
||||
fallback: NormalizedFragmentV2["domain_relevance"]
|
||||
): NormalizedFragmentV2["domain_relevance"] {
|
||||
if (typeof value === "boolean") {
|
||||
return value ? "in_scope" : "out_of_scope";
|
||||
}
|
||||
const token = normalizeToken(value);
|
||||
if (DOMAIN_RELEVANCE_VALUES.includes(token as NormalizedFragmentV2["domain_relevance"])) {
|
||||
return token as NormalizedFragmentV2["domain_relevance"];
|
||||
}
|
||||
if (["in_scope_true", "in_scope_yes", "in_scope_relevant", "relevant", "supported"].includes(token)) {
|
||||
return "in_scope";
|
||||
}
|
||||
if (["out_scope", "outofscope", "offtopic", "off_topic", "irrelevant"].includes(token)) {
|
||||
return "out_of_scope";
|
||||
}
|
||||
if (token === "true") return "in_scope";
|
||||
if (token === "false") return "out_of_scope";
|
||||
if (["unknown", "ambiguous", "n_a", "na"].includes(token)) return "unclear";
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceBusinessScope(
|
||||
value: unknown,
|
||||
fallback: NormalizedFragmentV2["business_scope"],
|
||||
domainRelevance: NormalizedFragmentV2["domain_relevance"]
|
||||
): NormalizedFragmentV2["business_scope"] {
|
||||
const token = normalizeToken(value);
|
||||
if (BUSINESS_SCOPE_VALUES.includes(token as NormalizedFragmentV2["business_scope"])) {
|
||||
return token as NormalizedFragmentV2["business_scope"];
|
||||
}
|
||||
if (["company_specific", "company_accounting", "document_review", "settlement", "bank_settlement"].includes(token)) {
|
||||
return "company_specific_accounting";
|
||||
}
|
||||
if (["generic", "general_accounting", "general"].includes(token)) {
|
||||
return "generic_accounting";
|
||||
}
|
||||
if (["out_of_scope", "off_topic", "outside"].includes(token)) {
|
||||
return "offtopic";
|
||||
}
|
||||
if (token === "unknown") {
|
||||
return "unclear";
|
||||
}
|
||||
if (domainRelevance === "out_of_scope") {
|
||||
return "offtopic";
|
||||
}
|
||||
if (domainRelevance === "in_scope") {
|
||||
return "company_specific_accounting";
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceFragmentId(value: unknown, index: number, fallback: string): string {
|
||||
if (typeof value === "string" && value.trim().length > 0) {
|
||||
return value.trim();
|
||||
}
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const n = Math.max(1, Math.floor(value));
|
||||
return `F${n}`;
|
||||
}
|
||||
return fallback || `F${index + 1}`;
|
||||
}
|
||||
|
||||
function parseYear(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isInteger(value) && value >= 1900 && value <= 2200) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
if (/^\d{4}$/.test(trimmed)) {
|
||||
const parsed = Number.parseInt(trimmed, 10);
|
||||
if (Number.isInteger(parsed) && parsed >= 1900 && parsed <= 2200) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseMonth(value: unknown): number | null {
|
||||
if (typeof value === "number" && Number.isInteger(value) && value >= 1 && value <= 12) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const trimmed = value.trim();
|
||||
if (/^\d{1,2}$/.test(trimmed)) {
|
||||
const parsed = Number.parseInt(trimmed, 10);
|
||||
if (parsed >= 1 && parsed <= 12) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function coerceTimeScope(
|
||||
value: unknown,
|
||||
rawText: string,
|
||||
fallback: NormalizedFragmentV2["time_scope"]
|
||||
): NormalizedFragmentV2["time_scope"] {
|
||||
if (value && typeof value === "object") {
|
||||
const source = value as Record<string, unknown>;
|
||||
const rawType = normalizeToken(source.type);
|
||||
const confidence = coerceConfidence(source.confidence, fallback.confidence);
|
||||
if (["explicit", "inferred", "missing"].includes(rawType)) {
|
||||
if (rawType === "missing") {
|
||||
return {
|
||||
type: "missing",
|
||||
value: null,
|
||||
confidence
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: rawType as NormalizedFragmentV2["time_scope"]["type"],
|
||||
value: toOptionalString(source.value),
|
||||
confidence
|
||||
};
|
||||
}
|
||||
|
||||
const periodType = normalizeToken(source.period_type);
|
||||
const year = parseYear(source.year);
|
||||
const month = parseMonth(source.month);
|
||||
if ((periodType === "year" || (periodType.length === 0 && year !== null)) && year !== null) {
|
||||
return {
|
||||
type: "explicit",
|
||||
value: String(year),
|
||||
confidence: confidence === "low" ? "medium" : confidence
|
||||
};
|
||||
}
|
||||
if ((periodType === "month" || periodType === "year_month" || (year !== null && month !== null)) && year !== null && month !== null) {
|
||||
return {
|
||||
type: "explicit",
|
||||
value: `${year}-${String(month).padStart(2, "0")}`,
|
||||
confidence: confidence === "low" ? "medium" : confidence
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const inferred = inferTimeScope(rawText);
|
||||
if (inferred.type !== "missing") {
|
||||
return inferred;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceFlags(
|
||||
value: unknown,
|
||||
fallback: NormalizedFragmentV2["flags"]
|
||||
): NormalizedFragmentV2["flags"] {
|
||||
if (!value || typeof value !== "object") {
|
||||
return fallback;
|
||||
}
|
||||
const source = value as Record<string, unknown>;
|
||||
const pick = (key: keyof NormalizedFragmentV2["flags"], aliases: string[] = []): boolean => {
|
||||
if (key in source) {
|
||||
return coerceBoolean(source[key], fallback[key]);
|
||||
}
|
||||
for (const alias of aliases) {
|
||||
if (alias in source) {
|
||||
return coerceBoolean(source[alias], fallback[key]);
|
||||
}
|
||||
}
|
||||
return fallback[key];
|
||||
};
|
||||
|
||||
return {
|
||||
has_multi_entity_scope: pick("has_multi_entity_scope", ["multi_entity_scope"]),
|
||||
asks_for_chain_explanation: pick("asks_for_chain_explanation", ["asks_for_chain", "chain_explanation"]),
|
||||
asks_for_ranking_or_top: pick("asks_for_ranking_or_top", ["asks_for_ranking", "asks_for_top"]),
|
||||
asks_for_period_summary: pick("asks_for_period_summary", ["period_summary"]),
|
||||
asks_for_rule_check: pick("asks_for_rule_check", ["rule_check"]),
|
||||
asks_for_anomaly_scan: pick("asks_for_anomaly_scan", ["anomaly_scan"]),
|
||||
asks_for_exact_object_trace: pick("asks_for_exact_object_trace", ["exact_object_trace"]),
|
||||
asks_for_evidence: pick("asks_for_evidence", ["evidence"]),
|
||||
mentions_period_close_context: pick("mentions_period_close_context", ["period_close_context"])
|
||||
};
|
||||
}
|
||||
|
||||
function mapCandidateLabel(value: string): NormalizedFragmentV2["candidate_labels"][number] | null {
|
||||
const token = normalizeToken(value);
|
||||
if (CANDIDATE_LABEL_VALUES.includes(token as NormalizedFragmentV2["candidate_labels"][number])) {
|
||||
return token as NormalizedFragmentV2["candidate_labels"][number];
|
||||
}
|
||||
if (["show_documents", "document_list", "show_docs", "point_answer", "lookup"].includes(token)) {
|
||||
return "simple_factual";
|
||||
}
|
||||
if (["ranking", "top", "summary", "analytical"].includes(token)) {
|
||||
return "heavy_analytical";
|
||||
}
|
||||
if (["chain", "cross", "cross_domain"].includes(token)) {
|
||||
return "cross_entity";
|
||||
}
|
||||
if (["rule_check", "control", "rules"].includes(token)) {
|
||||
return "rule_based_account_control";
|
||||
}
|
||||
if (["risk_scan", "anomaly", "risk"].includes(token)) {
|
||||
return "anomaly_probe";
|
||||
}
|
||||
if (["period_close", "month_close"].includes(token)) {
|
||||
return "period_close_risk";
|
||||
}
|
||||
if (["ambiguous", "unclear"].includes(token)) {
|
||||
return "ambiguous_human_query";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function coerceCandidateLabels(
|
||||
value: unknown,
|
||||
flags: NormalizedFragmentV2["flags"],
|
||||
domainRelevance: NormalizedFragmentV2["domain_relevance"],
|
||||
fallback: NormalizedFragmentV2["candidate_labels"]
|
||||
): NormalizedFragmentV2["candidate_labels"] {
|
||||
const parsed = coerceStringArray(value)
|
||||
.map((item) => mapCandidateLabel(item))
|
||||
.filter((item): item is NormalizedFragmentV2["candidate_labels"][number] => Boolean(item));
|
||||
if (parsed.length > 0) {
|
||||
return Array.from(new Set(parsed));
|
||||
}
|
||||
const inferred = pickCandidateLabels(flags, domainRelevance);
|
||||
if (inferred.length > 0) {
|
||||
return inferred;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function coerceFragmentV2(rawFragment: unknown, index: number, userMessage: string): NormalizedFragmentV2 | null {
|
||||
const source = rawFragment && typeof rawFragment === "object" ? (rawFragment as Record<string, unknown>) : {};
|
||||
const rawText =
|
||||
toOptionalString(source.raw_fragment_text) ??
|
||||
toOptionalString(source.rawText) ??
|
||||
toOptionalString(source.fragment_text) ??
|
||||
toOptionalString(source.text) ??
|
||||
userMessage.trim();
|
||||
const base = buildFragmentV2(rawText, index) ?? buildFragmentV2(userMessage, index);
|
||||
if (!base) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const domainRelevance = coerceDomainRelevance(source.domain_relevance, base.domain_relevance);
|
||||
const businessScope = coerceBusinessScope(source.business_scope, base.business_scope, domainRelevance);
|
||||
const flags = coerceFlags(source.flags, base.flags);
|
||||
|
||||
const entityHints = coerceStringArray(source.entity_hints);
|
||||
const accountHints = coerceStringArray(source.account_hints);
|
||||
const documentHints = coerceStringArray(source.document_hints);
|
||||
const registerHints = coerceStringArray(source.register_hints);
|
||||
|
||||
return {
|
||||
fragment_id: coerceFragmentId(source.fragment_id, index, base.fragment_id),
|
||||
raw_fragment_text: rawText,
|
||||
normalized_fragment_text: toOptionalString(source.normalized_fragment_text) ?? base.normalized_fragment_text,
|
||||
domain_relevance: domainRelevance,
|
||||
business_scope: businessScope,
|
||||
entity_hints: entityHints.length > 0 ? entityHints : base.entity_hints,
|
||||
account_hints: accountHints.length > 0 ? accountHints : base.account_hints,
|
||||
document_hints: documentHints.length > 0 ? documentHints : base.document_hints,
|
||||
register_hints: registerHints.length > 0 ? registerHints : base.register_hints,
|
||||
time_scope: coerceTimeScope(source.time_scope, rawText, base.time_scope),
|
||||
flags,
|
||||
candidate_labels: coerceCandidateLabels(source.candidate_labels, flags, domainRelevance, base.candidate_labels),
|
||||
confidence: coerceConfidence(source.confidence, base.confidence)
|
||||
};
|
||||
}
|
||||
|
||||
function coerceDiscardedFragments(value: unknown): DiscardedFragmentV2[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
const collected: DiscardedFragmentV2[] = [];
|
||||
for (const item of value) {
|
||||
if (!item || typeof item !== "object") {
|
||||
continue;
|
||||
}
|
||||
const source = item as Record<string, unknown>;
|
||||
const raw = toOptionalString(source.raw_fragment_text);
|
||||
const reason = toOptionalString(source.reason);
|
||||
if (!raw || !reason) {
|
||||
continue;
|
||||
}
|
||||
collected.push({
|
||||
raw_fragment_text: raw,
|
||||
reason
|
||||
});
|
||||
}
|
||||
return collected;
|
||||
}
|
||||
|
||||
function coerceScopeConfidence(
|
||||
value: unknown,
|
||||
fallback: NormalizedQueryV2["scope_confidence"]
|
||||
): NormalizedQueryV2["scope_confidence"] {
|
||||
return coerceConfidence(value, fallback);
|
||||
}
|
||||
|
||||
function coerceGlobalNotes(
|
||||
value: unknown,
|
||||
fallbackNeedsClarification: boolean
|
||||
): NormalizedQueryV2["global_notes"] {
|
||||
if (!value || typeof value !== "object") {
|
||||
return {
|
||||
needs_clarification: fallbackNeedsClarification,
|
||||
clarification_reason: fallbackNeedsClarification ? "clarification_required" : null
|
||||
};
|
||||
}
|
||||
const source = value as Record<string, unknown>;
|
||||
const needs = coerceBoolean(source.needs_clarification, fallbackNeedsClarification);
|
||||
const clarificationReason = toOptionalString(source.clarification_reason);
|
||||
return {
|
||||
needs_clarification: needs,
|
||||
clarification_reason: needs ? clarificationReason ?? "clarification_required" : null
|
||||
};
|
||||
}
|
||||
|
||||
function coerceNormalizedCandidateV2(candidate: unknown, userMessage: string): NormalizedQueryV2 | null {
|
||||
if (!candidate || typeof candidate !== "object") {
|
||||
return null;
|
||||
}
|
||||
const source = candidate as Record<string, unknown>;
|
||||
const sourceFragments = Array.isArray(source.fragments)
|
||||
? source.fragments
|
||||
: source.fragment && typeof source.fragment === "object"
|
||||
? [source.fragment]
|
||||
: splitIntoCandidateFragments(userMessage).map((text) => ({ raw_fragment_text: text }));
|
||||
|
||||
const fragments = sourceFragments
|
||||
.map((item, index) => coerceFragmentV2(item, index, userMessage))
|
||||
.filter((item): item is NormalizedFragmentV2 => item !== null);
|
||||
|
||||
const inScopeCount = fragments.filter((item) => item.domain_relevance === "in_scope").length;
|
||||
const unclearCount = fragments.filter((item) => item.domain_relevance === "unclear").length;
|
||||
const messageInScope = inScopeCount > 0;
|
||||
const inferredScopeConfidence: NormalizedQueryV2["scope_confidence"] = messageInScope ? (unclearCount > 0 ? "medium" : "high") : "low";
|
||||
const inferredNeedsClarification = messageInScope && (unclearCount > 0 || fragments.some((item) => item.time_scope.type === "missing"));
|
||||
|
||||
return {
|
||||
schema_version: "normalized_query_v2",
|
||||
user_message_raw: toOptionalString(source.user_message_raw) ?? userMessage,
|
||||
message_in_scope: coerceBoolean(source.message_in_scope, messageInScope),
|
||||
scope_confidence: coerceScopeConfidence(source.scope_confidence, inferredScopeConfidence),
|
||||
contains_multiple_tasks: coerceBoolean(source.contains_multiple_tasks, fragments.length > 1),
|
||||
fragments,
|
||||
discarded_fragments: coerceDiscardedFragments(source.discarded_fragments),
|
||||
global_notes: coerceGlobalNotes(source.global_notes, inferredNeedsClarification)
|
||||
};
|
||||
}
|
||||
|
||||
function collectDateSpans(text: string): Array<{ start: number; end: number }> {
|
||||
const spans: Array<{ start: number; end: number }> = [];
|
||||
const patterns = [
|
||||
@@ -1099,6 +1541,7 @@ export class NormalizerService {
|
||||
try {
|
||||
normalizedCandidate = safeJsonParse(outputText);
|
||||
if (schemaVersion !== "v1") {
|
||||
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
|
||||
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
|
||||
}
|
||||
if (schemaVersion === "v2_0_2") {
|
||||
@@ -1150,6 +1593,7 @@ export class NormalizerService {
|
||||
try {
|
||||
normalizedCandidate = safeJsonParse(outputText);
|
||||
if (schemaVersion !== "v1") {
|
||||
normalizedCandidate = coerceNormalizedCandidateV2(normalizedCandidate, payload.userQuestion) ?? normalizedCandidate;
|
||||
normalizedCandidate = applyCompanyScopeResolutionV2(normalizedCandidate, payload.userQuestion, payload.context);
|
||||
}
|
||||
if (schemaVersion === "v2_0_2") {
|
||||
|
||||
Reference in New Issue
Block a user