Починить восстановление кириллицы в автопрогонах

llm_normalizer/backend/src/routes/autoRuns.ts

@@ -321,7 +321,7 @@ function parseAutoGenTitle(value: unknown): string | null {
   if (!title) {
     return null;
   }
-  return title.slice(0, 160);
+  return repairAutogenMojibake(title).slice(0, 160);
 }
 
 function parseManualCaseDecision(value: unknown, fallback: ManualCaseDecision = "needs_dialog_policy_fix"): ManualCaseDecision {
@@ -1504,7 +1504,9 @@ function textMojibakeScore(value: string): number {
   const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
   const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
   const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
-  return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
+  const replacement = (source.match(/\uFFFD/g) ?? []).length;
+  const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
+  return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
 }
 
 function looksLikeMojibake(value: string): boolean {
@@ -1521,16 +1523,39 @@ function looksLikeMojibake(value: string): boolean {
   return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
 }
 
+function encodeWin1251MojibakeBytes(value: string): Buffer {
+  const chunks: Buffer[] = [];
+  for (const char of String(value ?? "")) {
+    const code = char.codePointAt(0) ?? 0;
+    if (code >= 0x80 && code <= 0x9f) {
+      chunks.push(Buffer.from([code]));
+      continue;
+    }
+    chunks.push(iconv.encode(char, "win1251"));
+  }
+  return Buffer.concat(chunks);
+}
+
+function decodeUtf8FromWin1251Mojibake(value: string): string {
+  return encodeWin1251MojibakeBytes(value).toString("utf8");
+}
+
+function repairKnownReplacementDamagedAutogenText(value: string): string {
+  return String(value ?? "")
+    .replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
+    .replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
+}
+
 function repairAutogenMojibake(value: string): string {
-  const source = String(value ?? "");
-  if (!looksLikeMojibake(source)) {
+  const source = repairKnownReplacementDamagedAutogenText(String(value ?? ""));
+  if (!looksLikeMojibake(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
     return source;
   }
   let candidate = source;
   for (let pass = 0; pass < 3; pass += 1) {
     let improved = false;
     try {
-      const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
+      const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
       if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
         candidate = fromWin1251;
         improved = true;
@@ -1547,11 +1572,16 @@ function repairAutogenMojibake(value: string): string {
     } catch {
       // ignore
     }
+    const repairedKnownText = repairKnownReplacementDamagedAutogenText(candidate);
+    if (repairedKnownText !== candidate) {
+      candidate = repairedKnownText;
+      improved = true;
+    }
     if (!improved) {
       break;
     }
   }
-  return candidate;
+  return repairKnownReplacementDamagedAutogenText(candidate);
 }
 
 function sanitizeGeneratedQuestion(value: string): string {
@@ -1775,7 +1805,8 @@ function extractQuestionsFromAutogenOutput(rawText: string): string[] {
 
 export const __autoRunsQuestionTestUtils = {
   splitQuestionCandidates,
-  extractQuestionsFromAutogenOutput
+  extractQuestionsFromAutogenOutput,
+  repairAutogenMojibake
 };
 
 async function generateQwenSeedQuestionsLive(input: {

llm_normalizer/backend/src/routes/eval.ts

@@ -7,6 +7,7 @@ import type { AppServices } from "../serverContext";
 import { ApiError, ok } from "../utils/http";
 import type { EvalRunMode, NormalizeRequestPayload } from "../types/normalizer";
 import type { EvalTarget } from "../types/assistantEval";
+import { repairAddressMojibakeText } from "../services/addressTextRepair";
 
 type EvalAsyncStatus = "queued" | "running" | "completed" | "failed" | "canceled";
 
@@ -67,7 +68,7 @@ function toArray(value: unknown): unknown[] {
 }
 
 function normalizeQuestionChunk(value: string): string {
-  return String(value ?? "")
+  return repairAddressMojibakeText(String(value ?? ""))
     .replace(/\r/g, " ")
     .replace(/\t/g, " ")
     .replace(/\s+/g, " ")
@@ -136,7 +137,7 @@ function normalizeRuntimeQuestionList(items: string[]): string[] {
 }
 
 function splitQuestionCandidate(raw: string): string[] {
-  const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
+  const normalized = repairAddressMojibakeText(String(raw ?? "")).replace(/\r/g, "\n").trim();
   if (!normalized) {
     return [];
   }
@@ -533,7 +534,8 @@ export function buildEvalRouter(services: AppServices): Router {
       }
       const questions = normalizeRuntimeQuestions(body.questions);
       const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
-      const scenarioTitle = toStringSafe(body.scenarioTitle);
+      const scenarioTitleRaw = toStringSafe(body.scenarioTitle);
+      const scenarioTitle = scenarioTitleRaw ? repairAddressMojibakeText(scenarioTitleRaw) : null;
 
       const jobId = `job-${nanoid(10)}`;
       const runId = `assistant-stage1-${nanoid(10)}`;