SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Починить восстановление кириллицы в автопрогонах

Browse Source
This commit is contained in:
dctouch
2026-05-10 08:38:52 +03:00
parent 9b02083493
commit 3be06b5f93
18 changed files with 618 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm_normalizer/backend/src/routes/autoRuns.ts
+38 -7
View File
@@ -321,7 +321,7 @@ function parseAutoGenTitle(value: unknown): string | null {
if (!title) {
return null;
}
return title.slice(0, 160);
return repairAutogenMojibake(title).slice(0, 160);
}
function parseManualCaseDecision(value: unknown, fallback: ManualCaseDecision = "needs_dialog_policy_fix"): ManualCaseDecision {
@@ -1504,7 +1504,9 @@ function textMojibakeScore(value: string): number {
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const replacement = (source.match(/\uFFFD/g) ?? []).length;
const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
}
function looksLikeMojibake(value: string): boolean {
@@ -1521,16 +1523,39 @@ function looksLikeMojibake(value: string): boolean {
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
}
function encodeWin1251MojibakeBytes(value: string): Buffer {
const chunks: Buffer[] = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv.encode(char, "win1251"));
}
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251Mojibake(value: string): string {
return encodeWin1251MojibakeBytes(value).toString("utf8");
}
function repairKnownReplacementDamagedAutogenText(value: string): string {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
function repairAutogenMojibake(value: string): string {
const source = String(value ?? "");
if (!looksLikeMojibake(source)) {
const source = repairKnownReplacementDamagedAutogenText(String(value ?? ""));
if (!looksLikeMojibake(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
return source;
}
let candidate = source;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -1547,11 +1572,16 @@ function repairAutogenMojibake(value: string): string {
} catch {
// ignore
}
const repairedKnownText = repairKnownReplacementDamagedAutogenText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedAutogenText(candidate);
}
function sanitizeGeneratedQuestion(value: string): string {
@@ -1775,7 +1805,8 @@ function extractQuestionsFromAutogenOutput(rawText: string): string[] {
export const __autoRunsQuestionTestUtils = {
splitQuestionCandidates,
extractQuestionsFromAutogenOutput
extractQuestionsFromAutogenOutput,
repairAutogenMojibake
};
async function generateQwenSeedQuestionsLive(input: {
llm_normalizer/backend/src/routes/eval.ts
+5 -3
View File
@@ -7,6 +7,7 @@ import type { AppServices } from "../serverContext";
import { ApiError, ok } from "../utils/http";
import type { EvalRunMode, NormalizeRequestPayload } from "../types/normalizer";
import type { EvalTarget } from "../types/assistantEval";
import { repairAddressMojibakeText } from "../services/addressTextRepair";
type EvalAsyncStatus = "queued" | "running" | "completed" | "failed" | "canceled";
@@ -67,7 +68,7 @@ function toArray(value: unknown): unknown[] {
}
function normalizeQuestionChunk(value: string): string {
return String(value ?? "")
return repairAddressMojibakeText(String(value ?? ""))
.replace(/\r/g, " ")
.replace(/\t/g, " ")
.replace(/\s+/g, " ")
@@ -136,7 +137,7 @@ function normalizeRuntimeQuestionList(items: string[]): string[] {
}
function splitQuestionCandidate(raw: string): string[] {
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
const normalized = repairAddressMojibakeText(String(raw ?? "")).replace(/\r/g, "\n").trim();
if (!normalized) {
return [];
}
@@ -533,7 +534,8 @@ export function buildEvalRouter(services: AppServices): Router {
}
const questions = normalizeRuntimeQuestions(body.questions);
const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
const scenarioTitle = toStringSafe(body.scenarioTitle);
const scenarioTitleRaw = toStringSafe(body.scenarioTitle);
const scenarioTitle = scenarioTitleRaw ? repairAddressMojibakeText(scenarioTitleRaw) : null;
const jobId = `job-${nanoid(10)}`;
const runId = `assistant-stage1-${nanoid(10)}`;
llm_normalizer/backend/src/services/addressTextRepair.ts
+53 -17
View File
@@ -1,17 +1,26 @@
import iconv from "iconv-lite";
const MOJIBAKE_CONTINUATION_CLASS =
"[\\u0080-\\u00bf\\u0401-\\u040f\\u0451-\\u045f\\u2018-\\u201e\\u2020-\\u2022\\u2013-\\u2014\\u2122\\u20ac]";
const MOJIBAKE_PAIR_PATTERN = new RegExp(`(?:[\\u0420\\u0421]${MOJIBAKE_CONTINUATION_CLASS})`, "gu");
function compactWhitespace(value: string): string {
return value.replace(/\s+/g, " ").trim();
}
function countMatches(value: string, pattern: RegExp): number {
return (String(value ?? "").match(pattern) ?? []).length;
}
function textMojibakeScore(value: string): number {
const source = String(value ?? "");
const cyrillic = (source.match(/[\u0400-\u04ff]/g) ?? []).length;
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
const replacement = (source.match(/[�]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Р“[Р-џ]|Р’[Р-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - replacement * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const cyrillic = countMatches(source, /[\u0400-\u04ff]/g);
const latin = countMatches(source, /[A-Za-z]/g);
const replacement = countMatches(source, /\uFFFD/g);
const c1Controls = countMatches(source, /[\u0080-\u009f]/g);
const pairMarkers = countMatches(source, MOJIBAKE_PAIR_PATTERN);
const doubleEncodedMarkers = countMatches(source, /(?:\u0420[\u00a0-\u00bf]\u0421|\u0413[\u0080-\u00bf]|\u00c3.|\u00c2.)/gu);
return cyrillic + latin - replacement * 8 - c1Controls * 5 - pairMarkers * 3 - doubleEncodedMarkers * 2;
}
function looksLikeAddressMojibake(value: string): boolean {
@@ -19,30 +28,51 @@ function looksLikeAddressMojibake(value: string): boolean {
if (!source.trim()) {
return false;
}
if (/[�]/.test(source)) {
if (/[\u0080-\u009f\uFFFD]/.test(source)) {
return true;
}
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
if (countMatches(source, MOJIBAKE_PAIR_PATTERN) >= 2) {
return true;
}
if ((source.match(/(?:Р“[Р-џ]|Р’[Р-џ]|Ã.|Â.)/gu) ?? []).length >= 2) {
return true;
return countMatches(source, /(?:\u0420[\u00a0-\u00bf]\u0421|\u0413[\u0080-\u00bf]|\u00c3.|\u00c2.)/gu) >= 2;
}
function encodeWin1251MojibakeBytes(value: string): Buffer {
const chunks: Buffer[] = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv.encode(char, "win1251"));
}
return false;
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251Mojibake(value: string): string {
return encodeWin1251MojibakeBytes(value).toString("utf8");
}
function repairKnownReplacementDamagedRussianText(value: string): string {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
export function repairAddressMojibakeText(value: string): string {
const source = String(value ?? "");
if (!looksLikeAddressMojibake(source)) {
return source;
const sourceWithKnownRepairs = repairKnownReplacementDamagedRussianText(source);
if (!looksLikeAddressMojibake(sourceWithKnownRepairs)) {
return sourceWithKnownRepairs;
}
let candidate = source;
let candidate = sourceWithKnownRepairs;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251Mojibake(candidate);
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -61,14 +91,20 @@ export function repairAddressMojibakeText(value: string): string {
// Ignore decode failures and keep the current candidate.
}
const repairedKnownText = repairKnownReplacementDamagedRussianText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedRussianText(candidate);
}
export function normalizeRussianComparableText(value: unknown): string {
return compactWhitespace(repairAddressMojibakeText(String(value ?? "")).toLowerCase()).replace(/ё/g, "е");
return compactWhitespace(repairAddressMojibakeText(String(value ?? "")).toLowerCase()).replace(/\u0451/g, "\u0435");
}
llm_normalizer/backend/src/services/assistantLivingModePolicy.ts
+4 -1
View File
@@ -279,7 +279,10 @@ export function createAssistantLivingModePolicy(deps: AssistantLivingModePolicyD
if (hasAffectiveReactionCue) {
return false;
}
return normalized.length <= 36 && !/[?]/.test(String(userMessage ?? ""));
const rawQuestionProbe = String(userMessage ?? "")
.replace(/\uFFFD\?/g, "\uFFFD")
.replace(/пїЅ\?/giu, "пїЅ");
return normalized.length <= 36 && !/[?]/.test(rawQuestionProbe);
}
function hasAssistantDataScopeMetaQuestionSignal(text) {
llm_normalizer/backend/src/services/assistantOrganizationMatcher.ts
+9 -2
View File
@@ -189,18 +189,25 @@ export function scoreOrganizationMentionInMessage(message: unknown, organization
variantScore = Math.max(variantScore, variant.length * 5);
continue;
}
let fuzzyTokenScore = 0;
const fuzzyMatched = messageTokens.some((messageToken) => {
if (messageToken === variant) {
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * 5);
return true;
}
if (messageToken.length >= 5 && variant.length >= 5) {
return messageToken.startsWith(variant) || variant.startsWith(messageToken);
const prefixMatched = messageToken.startsWith(variant) || variant.startsWith(messageToken);
if (prefixMatched) {
const prefixLength = Math.min(messageToken.length, variant.length);
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * (prefixLength >= 7 ? 5 : 3));
}
return prefixMatched;
}
return false;
});
if (fuzzyMatched) {
matched = true;
variantScore = Math.max(variantScore, Math.max(20, variant.length * 3));
variantScore = Math.max(variantScore, Math.max(20, fuzzyTokenScore || variant.length * 3));
}
}
if (matched) {
llm_normalizer/backend/src/services/assistantService.ts
+41 -7
View File
@@ -1900,7 +1900,9 @@ function textMojibakeScoreForAddress(value) {
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ\uFFFD?’“”•–—™љ›њќћџ]/g) ?? []).length;
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
const replacement = (source.match(/\uFFFD/g) ?? []).length;
const c1Controls = (source.match(/[\u0080-\u009f]/g) ?? []).length;
return cyrillic + latin - replacement * 8 - c1Controls * 5 - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
}
function looksLikeMojibakeForAddress(value) {
const source = String(value ?? "");
@@ -1918,16 +1920,36 @@ function looksLikeMojibakeForAddress(value) {
}
return false;
}
function encodeWin1251MojibakeBytesForAddress(value) {
const chunks = [];
for (const char of String(value ?? "")) {
const code = char.codePointAt(0) ?? 0;
if (code >= 0x80 && code <= 0x9f) {
chunks.push(Buffer.from([code]));
continue;
}
chunks.push(iconv.encode(char, "win1251"));
}
return Buffer.concat(chunks);
}
function decodeUtf8FromWin1251MojibakeForAddress(value) {
return encodeWin1251MojibakeBytesForAddress(value).toString("utf8");
}
function repairKnownReplacementDamagedAddressText(value) {
return String(value ?? "")
.replace(/\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422[\uFFFD?]+\u0412\u0410/giu, "\u0410\u041b\u042c\u0422\u0415\u0420\u041d\u0410\u0422\u0418\u0412\u0410")
.replace(/\u041e\u0411\u0429[\uFFFD?]+\u0419/giu, "\u041e\u0411\u0429\u0418\u0419");
}
function repairAddressMojibake(value) {
const source = String(value ?? "");
if (!looksLikeMojibakeForAddress(source)) {
const source = repairKnownReplacementDamagedAddressText(String(value ?? ""));
if (!looksLikeMojibakeForAddress(source) && !/[\u0080-\u009f\uFFFD]/.test(source)) {
return source;
}
let candidate = source;
for (let pass = 0; pass < 3; pass += 1) {
let improved = false;
try {
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
const fromWin1251 = decodeUtf8FromWin1251MojibakeForAddress(candidate);
if (textMojibakeScoreForAddress(fromWin1251) > textMojibakeScoreForAddress(candidate)) {
candidate = fromWin1251;
improved = true;
@@ -1942,11 +1964,16 @@ function repairAddressMojibake(value) {
}
}
catch (_error) { }
const repairedKnownText = repairKnownReplacementDamagedAddressText(candidate);
if (repairedKnownText !== candidate) {
candidate = repairedKnownText;
improved = true;
}
if (!improved) {
break;
}
}
return candidate;
return repairKnownReplacementDamagedAddressText(candidate);
}
function sanitizeOutgoingAssistantText(value, fallback = "Не смог сформировать читаемый ответ. Уточните запрос.") {
const repaired = repairAddressMojibake(String(value ?? ""));
@@ -4268,18 +4295,25 @@ function scoreOrganizationMentionInMessage(message, organization) {
variantScore = Math.max(variantScore, variant.length * 5);
continue;
}
let fuzzyTokenScore = 0;
const fuzzyMatched = messageTokens.some((messageToken) => {
if (messageToken === variant) {
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * 5);
return true;
}
if (messageToken.length >= 5 && variant.length >= 5) {
return messageToken.startsWith(variant) || variant.startsWith(messageToken);
const prefixMatched = messageToken.startsWith(variant) || variant.startsWith(messageToken);
if (prefixMatched) {
const prefixLength = Math.min(messageToken.length, variant.length);
fuzzyTokenScore = Math.max(fuzzyTokenScore, variant.length * (prefixLength >= 7 ? 5 : 3));
}
return prefixMatched;
}
return false;
});
if (fuzzyMatched) {
matched = true;
variantScore = Math.max(variantScore, Math.max(20, variant.length * 3));
variantScore = Math.max(variantScore, Math.max(20, fuzzyTokenScore || variant.length * 3));
}
}
if (matched) {