SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ГЛОБАЛЬНЫЙ РЕФАКТОРИНГ АРХИТЕКТУРЫ - Рефакторинг этапов 2.2.1 - фикс деградаций по старым доменам + легкая доводка регрессий перед стартом 3его этапа

Browse Source
This commit is contained in:
dctouch
2026-04-11 16:19:53 +03:00
parent b5bd4fd737
commit 66402439dc
22 changed files with 2640 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm_normalizer/backend/dist/routes/autoRuns.js
Vendored
+75 -11
View File
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.__autoRunsQuestionTestUtils = void 0;
exports.buildAutoRunsRouter = buildAutoRunsRouter;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
@@ -1148,6 +1149,61 @@ function sanitizeGeneratedQuestion(value) {
.replace(/\s+/g, " ")
.trim();
}
const AUTOGEN_QUESTION_PLACEHOLDER_PATTERN = /^(?:questions?|вопросы?|список\s+вопросов)$/iu;
const AUTOGEN_QUESTION_TAIL_PATTERNS = [
/^(?:без\s+воды|по\s+факту|и\s+коротко|коротко|прям(?:\s+)?сейчас|за\s+весь\s+период|по\s+делу)\??$/iu
];
function stripAutogenQuestionSuffix(value) {
return sanitizeGeneratedQuestion(value).replace(/[?!.:,;]+$/u, "").trim();
}
function isAutogenQuestionPlaceholder(value) {
const core = stripAutogenQuestionSuffix(value).toLowerCase();
return core.length > 0 && AUTOGEN_QUESTION_PLACEHOLDER_PATTERN.test(core);
}
function isLikelyAutogenQuestionTail(value) {
const core = stripAutogenQuestionSuffix(value).toLowerCase();
if (!core) {
return false;
}
if (isAutogenQuestionPlaceholder(core)) {
return true;
}
return AUTOGEN_QUESTION_TAIL_PATTERNS.some((pattern) => pattern.test(core));
}
function mergeAutogenQuestionTail(baseQuestion, tail) {
const base = stripAutogenQuestionSuffix(baseQuestion);
const suffix = stripAutogenQuestionSuffix(tail);
if (!base) {
return suffix ? `${suffix}?` : "";
}
if (!suffix) {
return `${base}?`;
}
return `${base} ${suffix}?`
.replace(/\s+/g, " ")
.trim();
}
function normalizeAutogenQuestionCandidates(candidates) {
const normalized = [];
for (const candidate of candidates) {
const question = sanitizeGeneratedQuestion(candidate);
if (!question) {
continue;
}
if (isAutogenQuestionPlaceholder(question)) {
continue;
}
if (isLikelyAutogenQuestionTail(question) && normalized.length > 0) {
const merged = mergeAutogenQuestionTail(normalized[normalized.length - 1], question);
if (merged) {
normalized[normalized.length - 1] = merged;
}
continue;
}
normalized.push(question);
}
return normalized.filter((item) => item.length > 0);
}
function splitQuestionCandidates(rawText) {
const normalized = repairAutogenMojibake(rawText).replace(/\r/g, "\n").trim();
if (!normalized)
@@ -1159,27 +1215,30 @@ function splitQuestionCandidates(rawText) {
.map((line) => sanitizeGeneratedQuestion(line))
.filter((line) => line.length > 0);
if (byLines.length > 1) {
return byLines;
return normalizeAutogenQuestionCandidates(byLines);
}
const questionMarkCount = (unescaped.match(/\?/g) ?? []).length;
if (questionMarkCount > 1) {
const byQuestion = unescaped
.split("?")
.map((chunk) => sanitizeGeneratedQuestion(chunk))
.filter((chunk) => chunk.length > 0)
.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`));
if (byQuestion.length > 1) {
return byQuestion;
const questionChunks = Array.from(unescaped.matchAll(/[^?]+(?:\?|$)/g))
.map((match) => sanitizeGeneratedQuestion(match[0]))
.filter((chunk) => chunk.length > 0);
if (questionChunks.length > 1) {
const canSafelySplit = questionChunks.every((chunk) => !isAutogenQuestionPlaceholder(chunk) &&
!isLikelyAutogenQuestionTail(chunk) &&
sanitizeGeneratedQuestion(chunk).length >= 18);
if (canSafelySplit) {
return normalizeAutogenQuestionCandidates(questionChunks.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`)));
}
}
}
const quoted = Array.from(unescaped.matchAll(/"([^"\n]{6,}?)"/g))
.map((match) => sanitizeGeneratedQuestion(match[1]))
.filter((line) => line.length > 0);
if (quoted.length > 1) {
return quoted;
return normalizeAutogenQuestionCandidates(quoted);
}
const cleaned = sanitizeGeneratedQuestion(unescaped);
return cleaned ? [cleaned] : [];
return cleaned ? normalizeAutogenQuestionCandidates([cleaned]) : [];
}
function parseAutogenOutputJson(rawText) {
const cleaned = repairAutogenMojibake(rawText)
@@ -1225,7 +1284,8 @@ function collectQuestionsFromCandidate(value, depth = 0) {
return [];
}
if (Array.isArray(value)) {
return value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
const expanded = value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
return normalizeAutogenQuestionCandidates(expanded);
}
if (typeof value === "string") {
const text = value.trim();
@@ -1271,6 +1331,10 @@ function extractQuestionsFromAutogenOutput(rawText) {
}
return collectQuestionsFromCandidate(rawText);
}
exports.__autoRunsQuestionTestUtils = {
splitQuestionCandidates,
extractQuestionsFromAutogenOutput
};
async function generateQwenSeedQuestionsLive(input) {
const seedExamples = collectCanonicalQuestions(40);
const fallbackExamples = fallbackDomainTemplates(input.domain);
llm_normalizer/backend/dist/routes/eval.js
Vendored
+77 -5
View File
@@ -3,6 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.__evalRouteTestUtils = void 0;
exports.buildEvalRouter = buildEvalRouter;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
@@ -35,6 +36,61 @@ function normalizeQuestionChunk(value) {
.replace(/\s+/g, " ")
.trim();
}
const RUNTIME_QUESTION_PLACEHOLDER_PATTERN = /^(?:questions?|вопросы?|список\s+вопросов)$/iu;
const RUNTIME_QUESTION_TAIL_PATTERNS = [
/^(?:без\s+воды|по\s+факту|и\s+коротко|коротко|прям(?:\s+)?сейчас|за\s+весь\s+период|по\s+делу)\??$/iu
];
function stripQuestionSuffix(value) {
return normalizeQuestionChunk(value).replace(/[?!.:,;]+$/u, "").trim();
}
function isRuntimeQuestionPlaceholder(value) {
const core = stripQuestionSuffix(value).toLowerCase();
return core.length > 0 && RUNTIME_QUESTION_PLACEHOLDER_PATTERN.test(core);
}
function isLikelyRuntimeQuestionTail(value) {
const core = stripQuestionSuffix(value).toLowerCase();
if (!core) {
return false;
}
if (isRuntimeQuestionPlaceholder(core)) {
return true;
}
return RUNTIME_QUESTION_TAIL_PATTERNS.some((pattern) => pattern.test(core));
}
function mergeRuntimeQuestionTail(baseQuestion, tail) {
const base = stripQuestionSuffix(baseQuestion);
const suffix = stripQuestionSuffix(tail);
if (!base) {
return suffix ? `${suffix}?` : "";
}
if (!suffix) {
return `${base}?`;
}
return `${base} ${suffix}?`
.replace(/\s+/g, " ")
.trim();
}
function normalizeRuntimeQuestionList(items) {
const normalized = [];
for (const item of items) {
const chunk = normalizeQuestionChunk(item);
if (!chunk) {
continue;
}
if (isRuntimeQuestionPlaceholder(chunk)) {
continue;
}
if (isLikelyRuntimeQuestionTail(chunk) && normalized.length > 0) {
const merged = mergeRuntimeQuestionTail(normalized[normalized.length - 1], chunk);
if (merged) {
normalized[normalized.length - 1] = merged;
}
continue;
}
normalized.push(chunk);
}
return normalized.filter((item) => item.length > 0);
}
function splitQuestionCandidate(raw) {
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
if (!normalized) {
@@ -47,18 +103,30 @@ function splitQuestionCandidate(raw) {
const source = byLines.length > 1 ? byLines : [normalized];
const chunks = [];
for (const line of source) {
const normalizedLine = normalizeQuestionChunk(line);
if (!normalizedLine || isRuntimeQuestionPlaceholder(normalizedLine)) {
continue;
}
const questionLike = Array.from(line.matchAll(/[^?]+(?:\?|$)/g))
.map((match) => normalizeQuestionChunk(match[0]))
.filter((item) => item.length > 0);
if (questionLike.length > 1) {
for (const item of questionLike) {
chunks.push(item.endsWith("?") ? item : `${item}?`);
const canSafelySplit = questionLike.every((item) => !isRuntimeQuestionPlaceholder(item) &&
!isLikelyRuntimeQuestionTail(item) &&
normalizeQuestionChunk(item).length >= 18);
if (canSafelySplit) {
for (const item of questionLike) {
chunks.push(item.endsWith("?") ? item : `${item}?`);
}
}
else {
chunks.push(normalizedLine);
}
continue;
}
chunks.push(normalizeQuestionChunk(line));
chunks.push(normalizedLine);
}
return chunks.filter((item) => item.length > 0);
return normalizeRuntimeQuestionList(chunks);
}
function normalizeRuntimeQuestions(value) {
const raw = toArray(value)
@@ -67,7 +135,7 @@ function normalizeRuntimeQuestions(value) {
if (raw.length === 0) {
return [];
}
const expanded = raw.flatMap((item) => splitQuestionCandidate(item));
const expanded = normalizeRuntimeQuestionList(raw.flatMap((item) => splitQuestionCandidate(item)));
const deduped = [];
const seen = new Set();
for (const item of expanded) {
@@ -81,6 +149,10 @@ function normalizeRuntimeQuestions(value) {
}
return deduped;
}
exports.__evalRouteTestUtils = {
splitQuestionCandidate,
normalizeRuntimeQuestions
};
function normalizeCaseIds(value) {
if (!Array.isArray(value)) {
return undefined;