Починить восстановление кириллицы в автопрогонах

llm_normalizer/backend/dist/routes/autoRuns.js

@@ -102,7 +102,7 @@ function parseAutoGenTitle(value) {
     if (!title) {
         return null;
     }
-    return title.slice(0, 160);
+    return repairAutogenMojibake(title).slice(0, 160);
 }
 function parseManualCaseDecision(value, fallback = "needs_dialog_policy_fix") {
     const normalized = toStringSafe(value);
@@ -1186,7 +1186,9 @@ function textMojibakeScore(value) {
     const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
     const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
     const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
-    return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
+    const replacement = (source.match(/\uFFFD/g) ?? []).length;
+    const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
+    return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
 }
 function looksLikeMojibake(value) {
     const source = String(value ?? "");
@@ -1201,16 +1203,36 @@ function looksLikeMojibake(value) {
     }
     return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
 }
+function encodeWin1251MojibakeBytes(value) {
+    const chunks = [];
+    for (const char of String(value ?? "")) {
+        const code = char.codePointAt(0) ?? 0;
+        if (code >= 0x80 && code <= 0x9f) {
+            chunks.push(Buffer.from([code]));
+            continue;
+        }
+        chunks.push(iconv_lite_1.default.encode(char, "win1251"));
+    }
+    return Buffer.concat(chunks);
+}
+function decodeUtf8FromWin1251Mojibake(value) {
+    return encodeWin1251MojibakeBytes(value).toString("utf8");
+}
+function repairKnownReplacementDamagedAutogenText(value) {
+    return String(value ?? "")
+        .replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
+        .replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
+}
 function repairAutogenMojibake(value) {
-    const source = String(value ?? "");
-    if (!looksLikeMojibake(source)) {
+    const source = repairKnownReplacementDamagedAutogenText(String(value ?? ""));
+    if (!looksLikeMojibake(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
         return source;
     }
     let candidate = source;
     for (let pass = 0; pass < 3; pass += 1) {
         let improved = false;
         try {
-            const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
+            const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
             if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
                 candidate = fromWin1251;
                 improved = true;
@@ -1229,11 +1251,16 @@ function repairAutogenMojibake(value) {
         catch {
             // ignore
         }
+        const repairedKnownText = repairKnownReplacementDamagedAutogenText(candidate);
+        if (repairedKnownText !== candidate) {
+            candidate = repairedKnownText;
+            improved = true;
+        }
         if (!improved) {
             break;
         }
     }
-    return candidate;
+    return repairKnownReplacementDamagedAutogenText(candidate);
 }
 function sanitizeGeneratedQuestion(value) {
     return repairAutogenMojibake(String(value ?? ""))
@@ -1432,7 +1459,8 @@ function extractQuestionsFromAutogenOutput(rawText) {
 }
 exports.__autoRunsQuestionTestUtils = {
     splitQuestionCandidates,
-    extractQuestionsFromAutogenOutput
+    extractQuestionsFromAutogenOutput,
+    repairAutogenMojibake
 };
 async function generateQwenSeedQuestionsLive(input) {
     const seedExamples = collectCanonicalQuestions(40);

llm_normalizer/backend/dist/routes/eval.js

@@ -11,6 +11,7 @@ const nanoid_1 = require("nanoid");
 const express_1 = require("express");
 const config_1 = require("../config");
 const http_1 = require("../utils/http");
+const addressTextRepair_1 = require("../services/addressTextRepair");
 const ASYNC_JOBS = new Map();
 const MAX_ASYNC_JOBS = 80;
 function toRecord(value) {
@@ -30,7 +31,7 @@ function toArray(value) {
     return Array.isArray(value) ? value : [];
 }
 function normalizeQuestionChunk(value) {
-    return String(value ?? "")
+    return (0, addressTextRepair_1.repairAddressMojibakeText)(String(value ?? ""))
         .replace(/\r/g, " ")
         .replace(/\t/g, " ")
         .replace(/\s+/g, " ")
@@ -92,7 +93,7 @@ function normalizeRuntimeQuestionList(items) {
     return normalized.filter((item) => item.length > 0);
 }
 function splitQuestionCandidate(raw) {
-    const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
+    const normalized = (0, addressTextRepair_1.repairAddressMojibakeText)(String(raw ?? "")).replace(/\r/g, "\n").trim();
     if (!normalized) {
         return [];
     }
@@ -455,7 +456,8 @@ function buildEvalRouter(services) {
             }
             const questions = normalizeRuntimeQuestions(body.questions);
             const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
-            const scenarioTitle = toStringSafe(body.scenarioTitle);
+            const scenarioTitleRaw = toStringSafe(body.scenarioTitle);
+            const scenarioTitle = scenarioTitleRaw ? (0, addressTextRepair_1.repairAddressMojibakeText)(scenarioTitleRaw) : null;
             const jobId = `job-${(0, nanoid_1.nanoid)(10)}`;
             const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
             const runtimeCaseSetFile = scenarioQuestions.length > 0