АДРЕСНЫЙ РЕЖИМ - ллм декомпоз
This commit is contained in:
@@ -13,6 +13,7 @@ import * as companyAnchorResolver_1 from "./companyAnchorResolver";
|
||||
import * as assistantRuntimeGuards_1 from "./assistantRuntimeGuards";
|
||||
import * as assistantClaimBoundEvidence_1 from "./assistantClaimBoundEvidence";
|
||||
import * as addressQueryService_1 from "./addressQueryService";
|
||||
import iconv from "iconv-lite";
|
||||
function retrievalSummaryForRoute(route) {
|
||||
if (route === "store_canonical")
|
||||
return "Canonical accounting data path selected.";
|
||||
@@ -1386,7 +1387,7 @@ function hasAccountingSignal(text) {
|
||||
}
|
||||
function hasFollowupMarker(text) {
|
||||
const compact = compactWhitespace(text.toLowerCase());
|
||||
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|plus|also|dobav|utochn|prodolzh)/i.test(compact);
|
||||
return /^(и|а еще|а ещё|еще|ещё|добав|уточн|продолж|также|а если|а теперь|теперь|plus|also|dobav|utochn|prodolzh|then|now)/i.test(compact);
|
||||
}
|
||||
function hasReferentialPointer(text) {
|
||||
return /(по этому|по тому|это же|этой|этим|этому|из этого|в этом|тот же|same thing|that one|po etomu|po tomu)/i.test(text.toLowerCase());
|
||||
@@ -1780,6 +1781,153 @@ function toNonEmptyString(value) {
|
||||
const text = String(value).trim();
|
||||
return text.length > 0 ? text : null;
|
||||
}
|
||||
const ADDRESS_PREDECOMPOSE_NOISE_TOKENS = new Set([
|
||||
"за",
|
||||
"с",
|
||||
"по",
|
||||
"на",
|
||||
"и",
|
||||
"или",
|
||||
"док",
|
||||
"доки",
|
||||
"docs",
|
||||
"documents",
|
||||
"doki",
|
||||
"dokument",
|
||||
"dokumenty",
|
||||
"документ",
|
||||
"документы",
|
||||
"документов",
|
||||
"банк",
|
||||
"банковские",
|
||||
"операции",
|
||||
"платеж",
|
||||
"платёж",
|
||||
"платежи",
|
||||
"контрагент",
|
||||
"контрагенту",
|
||||
"контрагента",
|
||||
"год",
|
||||
"года",
|
||||
"г",
|
||||
"year",
|
||||
"god",
|
||||
"плс",
|
||||
"pls",
|
||||
"пж",
|
||||
"пжлст",
|
||||
"пожалуйста",
|
||||
"please",
|
||||
"покеж",
|
||||
"покажи",
|
||||
"показать",
|
||||
"show",
|
||||
"list",
|
||||
"выведи",
|
||||
"которые",
|
||||
"какие",
|
||||
"какой",
|
||||
"есть",
|
||||
"est",
|
||||
"kakie",
|
||||
"kakoi",
|
||||
"vse",
|
||||
"all",
|
||||
"blya",
|
||||
"blyat",
|
||||
"епт",
|
||||
"ёпт",
|
||||
"бля"
|
||||
]);
|
||||
function textMojibakeScoreForAddress(value) {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2;
|
||||
}
|
||||
function looksLikeMojibakeForAddress(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2;
|
||||
}
|
||||
function repairAddressMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibakeForAddress(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
try {
|
||||
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
}
|
||||
}
|
||||
catch (_error) { }
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScoreForAddress(fromLatin1) > textMojibakeScoreForAddress(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
}
|
||||
}
|
||||
catch (_error) { }
|
||||
return candidate;
|
||||
}
|
||||
function extractAddressAnchorTokens(value) {
|
||||
const source = repairAddressMojibake(compactWhitespace(String(value ?? "").toLowerCase()));
|
||||
if (!source) {
|
||||
return [];
|
||||
}
|
||||
const tokens = source
|
||||
.split(/[^a-zа-яё0-9._-]+/iu)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length >= 2);
|
||||
const filtered = [];
|
||||
for (const token of tokens) {
|
||||
if (/^\d+$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:19|20)\d{2}$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:0?[1-9]|1[0-2])[./-](?:19|20)\d{2}$/.test(token) || /^(?:19|20)\d{2}[./-](?:0?[1-9]|1[0-2])$/.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (/^(?:янв|фев|мар|апр|май|июн|июл|авг|сен|сент|окт|ноя|дек|january|february|march|april|may|june|july|august|september|october|november|december)/i.test(token)) {
|
||||
continue;
|
||||
}
|
||||
if (ADDRESS_PREDECOMPOSE_NOISE_TOKENS.has(token)) {
|
||||
continue;
|
||||
}
|
||||
filtered.push(token);
|
||||
}
|
||||
return Array.from(new Set(filtered));
|
||||
}
|
||||
function selectPreferredAddressFragmentCandidate(rawText, normalizedText) {
|
||||
const normalizedCandidate = compactWhitespace(repairAddressMojibake(normalizedText ?? ""));
|
||||
const rawCandidate = compactWhitespace(repairAddressMojibake(rawText ?? ""));
|
||||
if (!normalizedCandidate && !rawCandidate) {
|
||||
return null;
|
||||
}
|
||||
if (!normalizedCandidate) {
|
||||
return rawCandidate;
|
||||
}
|
||||
if (!rawCandidate) {
|
||||
return normalizedCandidate;
|
||||
}
|
||||
const normalizedAnchors = extractAddressAnchorTokens(normalizedCandidate);
|
||||
const rawAnchors = extractAddressAnchorTokens(rawCandidate);
|
||||
if (rawAnchors.length > 0 && normalizedAnchors.length === 0) {
|
||||
return rawCandidate;
|
||||
}
|
||||
return normalizedCandidate;
|
||||
}
|
||||
function readAddressFilterString(addressDebug, key) {
|
||||
const filters = addressDebug?.extracted_filters;
|
||||
if (!filters || typeof filters !== "object") {
|
||||
@@ -1801,7 +1949,8 @@ function findLastAddressAssistantDebug(items) {
|
||||
return null;
|
||||
}
|
||||
function hasAddressFollowupContextSignal(userMessage) {
|
||||
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
|
||||
const repaired = repairAddressMojibake(String(userMessage ?? ""));
|
||||
const text = compactWhitespace(repaired.toLowerCase());
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
@@ -1811,10 +1960,16 @@ function hasAddressFollowupContextSignal(userMessage) {
|
||||
if (hasReferentialPointer(text)) {
|
||||
return true;
|
||||
}
|
||||
if (/(?:на\s+ту\s+же\s+дат[ауеы]|на\s+эту\s+же\s+дат[ауеы]|same\s+date|the\s+same\s+date|as\s+of\s+same\s+date)/iu.test(text)) {
|
||||
return true;
|
||||
}
|
||||
const shortFollowup = countTokens(text) <= 8;
|
||||
if (shortFollowup && hasFollowupMarker(text)) {
|
||||
return true;
|
||||
}
|
||||
if (shortFollowup && hasPeriodLiteral(text)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
function resolveAddressFollowupCarryoverContext(userMessage, items) {
|
||||
@@ -1851,11 +2006,12 @@ function resolveAddressFollowupCarryoverContext(userMessage, items) {
|
||||
};
|
||||
}
|
||||
function isAddressLlmPreDecomposeCandidate(userMessage) {
|
||||
const text = compactWhitespace(String(userMessage ?? "").toLowerCase());
|
||||
const repaired = repairAddressMojibake(String(userMessage ?? ""));
|
||||
const text = compactWhitespace(repaired.toLowerCase());
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|банк|выписк|платеж|оплат|поступлен|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?)/i.test(text);
|
||||
return /(?:\bдок\b|доки|документ|контрагент|договор|остаток|сч(?:е|ё)т|сальдо|банк|выписк|платеж|оплат|поступлен|поступлени|списан|реализац|сверк|взаиморасч|кто\s+должен|show|list|documents?|counterparty|contract|account|balance|bank\s+operations?|doki|dokument(?:y|ov|am|a)?|platezh|oplata|schet|saldo)/i.test(text);
|
||||
}
|
||||
function extractAddressQuestionFromNormalized(normalized) {
|
||||
if (!normalized || typeof normalized !== "object") {
|
||||
@@ -1873,13 +2029,16 @@ function extractAddressQuestionFromNormalized(normalized) {
|
||||
continue;
|
||||
}
|
||||
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
|
||||
if (readiness === "no_route") {
|
||||
continue;
|
||||
}
|
||||
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
|
||||
const rawText = toNonEmptyString(fragment.raw_fragment_text);
|
||||
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
|
||||
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
|
||||
if (!candidate) {
|
||||
continue;
|
||||
}
|
||||
if (candidate.length >= 3 && candidate.length <= 500) {
|
||||
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
|
||||
continue;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -1986,13 +2145,16 @@ function extractAddressQuestionFromRawNormalizerOutput(rawModelOutput) {
|
||||
continue;
|
||||
}
|
||||
const readiness = String(fragment.execution_readiness ?? "").trim().toLowerCase();
|
||||
if (readiness === "no_route") {
|
||||
continue;
|
||||
}
|
||||
const normalizedText = toNonEmptyString(fragment.normalized_fragment_text);
|
||||
const rawText = toNonEmptyString(fragment.raw_fragment_text);
|
||||
const candidate = compactWhitespace(normalizedText ?? rawText ?? "");
|
||||
const candidate = selectPreferredAddressFragmentCandidate(rawText ?? "", normalizedText ?? "");
|
||||
if (!candidate) {
|
||||
continue;
|
||||
}
|
||||
if (candidate.length >= 3 && candidate.length <= 500) {
|
||||
if (readiness === "no_route" && !isAddressLlmPreDecomposeCandidate(candidate)) {
|
||||
continue;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -2203,12 +2365,21 @@ export class AssistantService {
|
||||
reason: "disabled_by_feature_flag"
|
||||
};
|
||||
const addressInputMessage = toNonEmptyString(addressPreDecompose?.effectiveMessage) ?? userMessage;
|
||||
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
|
||||
const shouldPreferContextualLane = Boolean(carryover?.followupContext);
|
||||
if (shouldPreferContextualLane) {
|
||||
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
|
||||
followupContext: carryover.followupContext
|
||||
});
|
||||
if (contextualAddressLane?.handled) {
|
||||
return finalizeAddressLaneResponse(contextualAddressLane, addressInputMessage, carryover, addressPreDecompose);
|
||||
}
|
||||
}
|
||||
const primaryAddressLane = await this.addressQueryService.tryHandle(addressInputMessage);
|
||||
if (primaryAddressLane?.handled) {
|
||||
return finalizeAddressLaneResponse(primaryAddressLane, addressInputMessage, null, addressPreDecompose);
|
||||
}
|
||||
const carryover = resolveAddressFollowupCarryoverContext(userMessage, session.items);
|
||||
if (carryover?.followupContext) {
|
||||
if (!shouldPreferContextualLane && carryover?.followupContext) {
|
||||
const contextualAddressLane = await this.addressQueryService.tryHandle(addressInputMessage, {
|
||||
followupContext: carryover.followupContext
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user