SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Починить восстановление кириллицы в автопрогонах

Browse Source
This commit is contained in:
dctouch
2026-05-10 08:38:52 +03:00
parent 9b02083493
commit 3be06b5f93
18 changed files with 618 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm_normalizer/backend/dist/routes/autoRuns.js
Vendored
+35 -7
View File
@@ -102,7 +102,7 @@ function parseAutoGenTitle(value) {
if (!title) {
return null;
}
return title.slice(0, 160);
return repairAutogenMojibake(title).slice(0, 160);
}
function parseManualCaseDecision(value, fallback = "needs_dialog_policy_fix") {
const normalized = toStringSafe(value);
@@ -1186,7 +1186,9 @@ function textMojibakeScore(value) {
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const replacement = (source.match(/\uFFFD/g) ?? []).length;
const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
}
function looksLikeMojibake(value) {
const source = String(value ?? "");
@@ -1201,16 +1203,36 @@ function looksLikeMojibake(value) {
}
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
}
function encodeWin1251MojibakeBytes(value) {
const chunks = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv_lite_1.default.encode(char, "win1251"));
}
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251Mojibake(value) {
return encodeWin1251MojibakeBytes(value).toString("utf8");
}
function repairKnownReplacementDamagedAutogenText(value) {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
function repairAutogenMojibake(value) {
const source = String(value ?? "");
if (!looksLikeMojibake(source)) {
const source = repairKnownReplacementDamagedAutogenText(String(value ?? ""));
if (!looksLikeMojibake(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
return source;
}
let candidate = source;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -1229,11 +1251,16 @@ function repairAutogenMojibake(value) {
catch {
// ignore
}
const repairedKnownText = repairKnownReplacementDamagedAutogenText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedAutogenText(candidate);
}
function sanitizeGeneratedQuestion(value) {
return repairAutogenMojibake(String(value ?? ""))
@@ -1432,7 +1459,8 @@ function extractQuestionsFromAutogenOutput(rawText) {
}
exports.__autoRunsQuestionTestUtils = {
splitQuestionCandidates,
extractQuestionsFromAutogenOutput
extractQuestionsFromAutogenOutput,
repairAutogenMojibake
};
async function generateQwenSeedQuestionsLive(input) {
const seedExamples = collectCanonicalQuestions(40);
llm_normalizer/backend/dist/routes/eval.js
Vendored
+5 -3
View File
@@ -11,6 +11,7 @@ const nanoid_1 = require("nanoid");
const express_1 = require("express");
const config_1 = require("../config");
const http_1 = require("../utils/http");
const addressTextRepair_1 = require("../services/addressTextRepair");
const ASYNC_JOBS = new Map();
const MAX_ASYNC_JOBS = 80;
function toRecord(value) {
@@ -30,7 +31,7 @@ function toArray(value) {
return Array.isArray(value) ? value : [];
}
function normalizeQuestionChunk(value) {
return String(value ?? "")
return (0, addressTextRepair_1.repairAddressMojibakeText)(String(value ?? ""))
.replace(/\r/g, " ")
.replace(/\t/g, " ")
.replace(/\s+/g, " ")
@@ -92,7 +93,7 @@ function normalizeRuntimeQuestionList(items) {
return normalized.filter((item) => item.length > 0);
}
function splitQuestionCandidate(raw) {
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
const normalized = (0, addressTextRepair_1.repairAddressMojibakeText)(String(raw ?? "")).replace(/\r/g, "\n").trim();
if (!normalized) {
return [];
}
@@ -455,7 +456,8 @@ function buildEvalRouter(services) {
}
const questions = normalizeRuntimeQuestions(body.questions);
const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
const scenarioTitle = toStringSafe(body.scenarioTitle);
const scenarioTitleRaw = toStringSafe(body.scenarioTitle);
const scenarioTitle = scenarioTitleRaw ? (0, addressTextRepair_1.repairAddressMojibakeText)(scenarioTitleRaw) : null;
const jobId = `job-${(0, nanoid_1.nanoid)(10)}`;
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
const runtimeCaseSetFile = scenarioQuestions.length > 0
llm_normalizer/backend/dist/services/addressTextRepair.js
Vendored
+46 -17
View File
@@ -6,44 +6,68 @@ Object.defineProperty(exports, "__esModule", { value: true });
exports.repairAddressMojibakeText = repairAddressMojibakeText;
exports.normalizeRussianComparableText = normalizeRussianComparableText;
const iconv_lite_1 = __importDefault(require("iconv-lite"));
const MOJIBAKE_CONTINUATION_CLASS = "[\\u0080-\\u00bf\\u0401-\\u040f\\u0451-\\u045f\\u2018-\\u201e\\u2020-\\u2022\\u2013-\\u2014\\u2122\\u20ac]";
const MOJIBAKE_PAIR_PATTERN = new RegExp(`(?:[\\u0420\\u0421]${MOJIBAKE_CONTINUATION_CLASS})`, "gu");
function compactWhitespace(value) {
return value.replace(/\s+/g, " ").trim();
}
function countMatches(value, pattern) {
return (String(value ?? "").match(pattern) ?? []).length;
}
function textMojibakeScore(value) {
const source = String(value ?? "");
const cyrillic = (source.match(/[\u0400-\u04ff]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const replacement = (source.match(/[�]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Р“[Р-џ]|Р’[Р-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - replacement * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const cyrillic = countMatches(source, /[\u0400-\u04ff]/g);
const latin = countMatches(source, /[A-Za-z]/g);
const replacement = countMatches(source, /\uFFFD/g);
const c1Controls = countMatches(source, /[\u0080-\u009f]/g);
const pairMarkers = countMatches(source, MOJIBAKE_PAIR_PATTERN);
const doubleEncodedMarkers = countMatches(source, /(?:\u0420[\u00a0-\u00bf]\u0421|\u0413[\u0080-\u00bf]|\u00c3.|\u00c2.)/gu);
return cyrillic + latin - replacement * 8 - c1Controls * 5 - pairMarkers * 3 - doubleEncodedMarkers * 2;
}
function looksLikeAddressMojibake(value) {
const source = String(value ?? "");
if (!source.trim()) {
return false;
}
if (/[�]/.test(source)) {
if (/[\u0080-\u009f\uFFFD]/.test(source)) {
return true;
}
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
if (countMatches(source, MOJIBAKE_PAIR_PATTERN) >= 2) {
return true;
}
if ((source.match(/(?:Р“[Р-џ]|Р’[Р-џ]|Ã.|Â.)/gu) ?? []).length >= 2) {
return true;
return countMatches(source, /(?:\u0420[\u00a0-\u00bf]\u0421|\u0413[\u0080-\u00bf]|\u00c3.|\u00c2.)/gu) >= 2;
}
function encodeWin1251MojibakeBytes(value) {
const chunks = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv_lite_1.default.encode(char, "win1251"));
}
return false;
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251Mojibake(value) {
return encodeWin1251MojibakeBytes(value).toString("utf8");
}
function repairKnownReplacementDamagedRussianText(value) {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
function repairAddressMojibakeText(value) {
const source = String(value ?? "");
if (!looksLikeAddressMojibake(source)) {
return source;
const sourceWithKnownRepairs = repairKnownReplacementDamagedRussianText(source);
if (!looksLikeAddressMojibake(sourceWithKnownRepairs)) {
return sourceWithKnownRepairs;
}
let candidate = source;
let candidate = sourceWithKnownRepairs;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -62,12 +86,17 @@ function repairAddressMojibakeText(value) {
catch {
// Ignore decode failures and keep the current candidate.
}
const repairedKnownText = repairKnownReplacementDamagedRussianText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedRussianText(candidate);
}
function normalizeRussianComparableText(value) {
return compactWhitespace(repairAddressMojibakeText(String(value ?? "")).toLowerCase()).replace(/ё/g, "е");
return compactWhitespace(repairAddressMojibakeText(String(value ?? "")).toLowerCase()).replace(/\u0451/g, "\u0435");
}
llm_normalizer/backend/dist/services/assistantLivingModePolicy.js
Vendored
+4 -1
View File
@@ -209,7 +209,10 @@ function createAssistantLivingModePolicy(deps) {
if (hasAffectiveReactionCue) {
return false;
}
return normalized.length <= 36 && !/[?]/.test(String(userMessage ?? ""));
const rawQuestionProbe = String(userMessage ?? "")
.replace(/\uFFFD\?/g, "\uFFFD")
.replace(/пїЅ\?/giu, "пїЅ");
return normalized.length <= 36 && !/[?]/.test(rawQuestionProbe);
}
function hasAssistantDataScopeMetaQuestionSignal(text) {
const repaired = repairAddressMojibake(String(text ?? ""));
llm_normalizer/backend/dist/services/assistantOrganizationMatcher.js
Vendored
+9 -2
View File
@@ -182,18 +182,25 @@ function scoreOrganizationMentionInMessage(message, organization) {
variantScore = Math.max(variantScore, variant.length * 5);
continue;
}
let fuzzyTokenScore = 0;
const fuzzyMatched = messageTokens.some((messageToken) => {
if (messageToken === variant) {
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * 5);
return true;
}
if (messageToken.length >= 5 && variant.length >= 5) {
return messageToken.startsWith(variant) || variant.startsWith(messageToken);
const prefixMatched = messageToken.startsWith(variant) || variant.startsWith(messageToken);
if (prefixMatched) {
const prefixLength = Math.min(messageToken.length, variant.length);
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * (prefixLength >= 7 ? 5 : 3));
}
return prefixMatched;
}
return false;
});
if (fuzzyMatched) {
matched = true;
variantScore = Math.max(variantScore, Math.max(20, variant.length * 3));
variantScore = Math.max(variantScore, Math.max(20, fuzzyTokenScore || variant.length * 3));
}
}
if (matched) {
llm_normalizer/backend/dist/services/assistantService.js
Vendored
+41 -7
View File
@@ -1944,7 +1944,9 @@ function textMojibakeScoreForAddress(value) {
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ\uFFFD?’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const replacement = (source.match(/\uFFFD/g) ?? []).length;
const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
}
function looksLikeMojibakeForAddress(value) {
const source = String(value ?? "");
@@ -1962,16 +1964,36 @@ function looksLikeMojibakeForAddress(value) {
}
return false;
}
function encodeWin1251MojibakeBytesForAddress(value) {
const chunks = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv_lite_1.default.encode(char, "win1251"));
}
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251MojibakeForAddress(value) {
return encodeWin1251MojibakeBytesForAddress(value).toString("utf8");
}
function repairKnownReplacementDamagedAddressText(value) {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
function repairAddressMojibake(value) {
const source = String(value ?? "");
if (!looksLikeMojibakeForAddress(source)) {
const source = repairKnownReplacementDamagedAddressText(String(value ?? ""));
if (!looksLikeMojibakeForAddress(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
return source;
}
let candidate = source;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251MojibakeForAddress(candidate);
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -1986,11 +2008,16 @@ function repairAddressMojibake(value) {
}
}
catch (_error) { }
const repairedKnownText = repairKnownReplacementDamagedAddressText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedAddressText(candidate);
}
function sanitizeOutgoingAssistantText(value, fallback = "Не смог сформировать читаемый ответ. Уточните запрос.") {
const repaired = repairAddressMojibake(String(value ?? ""));
@@ -4311,18 +4338,25 @@ function scoreOrganizationMentionInMessage(message, organization) {
variantScore = Math.max(variantScore, variant.length * 5);
continue;
}
let fuzzyTokenScore = 0;
const fuzzyMatched = messageTokens.some((messageToken) => {
if (messageToken === variant) {
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * 5);
return true;
}
if (messageToken.length >= 5 && variant.length >= 5) {
return messageToken.startsWith(variant) || variant.startsWith(messageToken);
const prefixMatched = messageToken.startsWith(variant) || variant.startsWith(messageToken);
if (prefixMatched) {
const prefixLength = Math.min(messageToken.length, variant.length);
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * (prefixLength >= 7 ? 5 : 3));
}
return prefixMatched;
}
return false;
});
if (fuzzyMatched) {
matched = true;
variantScore = Math.max(variantScore, Math.max(20, variant.length * 3));
variantScore = Math.max(variantScore, Math.max(20, fuzzyTokenScore || variant.length * 3));
}
}
if (matched) {