АДРЕСНЫЙ РЕЖИМ - авторан история - юи + адресный рендер прогонов в реалтайме
This commit is contained in:
+466
-15
@@ -7,9 +7,11 @@ exports.buildAutoRunsRouter = buildAutoRunsRouter;
|
||||
const fs_1 = __importDefault(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const express_1 = require("express");
|
||||
const iconv_lite_1 = __importDefault(require("iconv-lite"));
|
||||
const config_1 = require("../config");
|
||||
const http_1 = require("../utils/http");
|
||||
const capabilitiesRegistry_1 = require("../services/capabilitiesRegistry");
|
||||
const openaiResponsesClient_1 = require("../services/openaiResponsesClient");
|
||||
const MANUAL_CASE_DECISIONS = [
|
||||
"covered_ok",
|
||||
"covered_but_bad_answer",
|
||||
@@ -102,6 +104,10 @@ function parseAnnotationAuthor(value) {
|
||||
return null;
|
||||
return author.slice(0, 80);
|
||||
}
|
||||
function parseAnnotationResolved(value, fallback = false) {
|
||||
const parsed = toBooleanSafe(value);
|
||||
return parsed === null ? fallback : parsed;
|
||||
}
|
||||
function readManualDecisionSchema() {
|
||||
const fallback = {
|
||||
schema_version: "manual_case_decision_schema_v1_fallback",
|
||||
@@ -150,6 +156,8 @@ function readAutoGenHistory() {
|
||||
questions: toArray(item.questions)
|
||||
.map((q) => toStringSafe(q))
|
||||
.filter((q) => q !== null)
|
||||
.map((q) => sanitizeGeneratedQuestion(q))
|
||||
.filter((q) => q.length > 0)
|
||||
.slice(0, 500),
|
||||
generated_by: toStringSafe(item.generated_by),
|
||||
saved_case_set_file: toStringSafe(item.saved_case_set_file),
|
||||
@@ -160,6 +168,12 @@ function readAutoGenHistory() {
|
||||
assistant_prompt_version: toStringSafe(toRecord(item.context)?.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(toRecord(item.context)?.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(toRecord(item.context)?.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
}))
|
||||
@@ -207,11 +221,11 @@ function collectCanonicalQuestions(limit = 300) {
|
||||
for (const testCase of cases) {
|
||||
const rawQuestion = toStringSafe(testCase.raw_question) ?? toStringSafe(testCase.user_message) ?? toStringSafe(testCase.query);
|
||||
if (rawQuestion) {
|
||||
questions.push(rawQuestion);
|
||||
questions.push(sanitizeGeneratedQuestion(rawQuestion));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(new Set(questions)).slice(0, limit);
|
||||
return Array.from(new Set(questions.filter((item) => item.length > 0))).slice(0, limit);
|
||||
}
|
||||
function normalizeDomainHint(value) {
|
||||
const domain = toStringSafe(value);
|
||||
@@ -219,6 +233,49 @@ function normalizeDomainHint(value) {
|
||||
return null;
|
||||
return domain.toLowerCase();
|
||||
}
|
||||
function buildAutogenPromptFromCapabilityGroup(group) {
|
||||
const supported = group.supported_operations.slice(0, 3).join(", ");
|
||||
const examples = group.typical_queries.slice(0, 2).join(" | ");
|
||||
const hints = group.one_c_hints.slice(0, 2).join(", ");
|
||||
const operationsPart = supported ? ` Опирайся на операции: ${supported}.` : "";
|
||||
const examplesPart = examples ? ` Ближайшие формулировки: ${examples}.` : "";
|
||||
const hintsPart = hints ? ` Можно мягко упоминать контекст 1С: ${hints}.` : "";
|
||||
return (`Генерируй реалистичные вопросы бухгалтера по группе "${group.group_title}".` +
|
||||
` Добавляй живую разговорную форму и опечатки, но сохраняй бизнес-смысл.${operationsPart}${examplesPart}${hintsPart}` +
|
||||
" Не выдумывай операции вне read-only режима.");
|
||||
}
|
||||
function buildAutogenPersonalityCatalog() {
|
||||
const builtIn = [
|
||||
{
|
||||
id: "general",
|
||||
label: "Общий контур",
|
||||
domain: null,
|
||||
default_prompt: "Генерируй реалистичные живые вопросы бухгалтера по 1С. Добавляй разговорные формулировки и опечатки, но сохраняй бизнес-смысл.",
|
||||
source: "built_in"
|
||||
}
|
||||
];
|
||||
const registry = (0, capabilitiesRegistry_1.loadCapabilitiesRegistry)();
|
||||
const registryBased = registry.groups.map((group) => ({
|
||||
id: `registry_${group.group_code}`,
|
||||
label: `${group.group_title} (реестр)`,
|
||||
domain: group.group_code,
|
||||
default_prompt: buildAutogenPromptFromCapabilityGroup(group),
|
||||
source: "capabilities_registry"
|
||||
}));
|
||||
const dedup = new Map();
|
||||
for (const item of [...builtIn, ...registryBased]) {
|
||||
if (!item.id.trim())
|
||||
continue;
|
||||
if (!dedup.has(item.id)) {
|
||||
dedup.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return [...dedup.values()].map((item) => ({
|
||||
...item,
|
||||
label: repairAutogenMojibake(item.label),
|
||||
default_prompt: repairAutogenMojibake(item.default_prompt)
|
||||
}));
|
||||
}
|
||||
function fallbackDomainTemplates(domain) {
|
||||
if (domain?.includes("vat") || domain?.includes("ндс")) {
|
||||
return [
|
||||
@@ -276,9 +333,9 @@ function generateQwenSeedQuestions(count, domain) {
|
||||
const out = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = bag[index % bag.length];
|
||||
out.push(mutateIntoQwenStyle(base, index));
|
||||
out.push(sanitizeGeneratedQuestion(mutateIntoQwenStyle(base, index)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
function generateCodexCreativeQuestions(count, domain) {
|
||||
const domainTemplates = fallbackDomainTemplates(domain);
|
||||
@@ -293,9 +350,9 @@ function generateCodexCreativeQuestions(count, domain) {
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = domainTemplates[index % domainTemplates.length];
|
||||
const pattern = patterns[index % patterns.length];
|
||||
out.push(pattern.replace("{q}", base));
|
||||
out.push(sanitizeGeneratedQuestion(pattern.replace("{q}", base)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
function generateAutogenId() {
|
||||
return `gen-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
@@ -325,6 +382,9 @@ function readAnnotations() {
|
||||
comment: toStringSafe(item.comment) ?? "",
|
||||
manual_case_decision: parseManualCaseDecision(item.manual_case_decision),
|
||||
annotation_author: parseAnnotationAuthor(item.annotation_author),
|
||||
resolved: parseAnnotationResolved(item.resolved),
|
||||
resolved_at: toStringSafe(item.resolved_at),
|
||||
resolved_by: parseAnnotationAuthor(item.resolved_by),
|
||||
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
|
||||
updated_at: toStringSafe(item.updated_at) ?? new Date().toISOString(),
|
||||
context: {
|
||||
@@ -334,7 +394,9 @@ function readAnnotations() {
|
||||
eval_target: toStringSafe(context?.eval_target) ?? "unknown",
|
||||
prompt_version: toStringSafe(context?.prompt_version),
|
||||
domain: toStringSafe(context?.domain),
|
||||
query_class: toStringSafe(context?.query_class)
|
||||
query_class: toStringSafe(context?.query_class),
|
||||
question_text: toStringSafe(context?.question_text),
|
||||
answer_text: toStringSafe(context?.answer_text)
|
||||
}
|
||||
};
|
||||
})
|
||||
@@ -946,6 +1008,37 @@ function withMessageAnnotations(runId, caseId, messages, annotations) {
|
||||
};
|
||||
});
|
||||
}
|
||||
function buildRunAggregateDialog(run, annotations) {
|
||||
const cases = buildCaseSummaries(run.report, run.run_id, false);
|
||||
const messages = [];
|
||||
const decomposition = [];
|
||||
let globalMessageIndex = 0;
|
||||
for (const item of cases) {
|
||||
const caseId = item.case_id;
|
||||
const caseDialog = loadSessionDialog(run.run_id, caseId) ?? buildFallbackDialog(run, caseId);
|
||||
const annotatedCaseMessages = withMessageAnnotations(run.run_id, caseId, caseDialog.messages, annotations);
|
||||
for (const caseMessage of annotatedCaseMessages) {
|
||||
const localMessageIndex = toNumberSafe(caseMessage.message_index) ?? 0;
|
||||
messages.push({
|
||||
...caseMessage,
|
||||
case_id: caseId,
|
||||
case_message_index: localMessageIndex,
|
||||
message_index: globalMessageIndex
|
||||
});
|
||||
globalMessageIndex += 1;
|
||||
}
|
||||
if (caseDialog.decomposition.length > 0) {
|
||||
decomposition.push(...caseDialog.decomposition.map((step) => `[${caseId}] ${step}`));
|
||||
}
|
||||
}
|
||||
return {
|
||||
source: "run_aggregate",
|
||||
session_id: `${run.run_id}::__all__`,
|
||||
messages,
|
||||
decomposition,
|
||||
assistant_mode: null
|
||||
};
|
||||
}
|
||||
function generateAnnotationId() {
|
||||
return `ann-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
}
|
||||
@@ -975,6 +1068,265 @@ function parseAutogenDomain(value) {
|
||||
return null;
|
||||
return domain.slice(0, 80);
|
||||
}
|
||||
function parseAutogenLlmRuntimeConfig(body, context) {
|
||||
const llm = toRecord(body.llm);
|
||||
const providerRaw = toStringSafe(llm?.llm_provider ?? context?.llm_provider)?.toLowerCase() ?? "";
|
||||
const model = toStringSafe(llm?.model ?? context?.model);
|
||||
if (!model || (providerRaw !== "openai" && providerRaw !== "local")) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
llm_provider: providerRaw === "local" ? "local" : "openai",
|
||||
api_key: toStringSafe(llm?.api_key) ?? "",
|
||||
model,
|
||||
base_url: toStringSafe(llm?.base_url),
|
||||
temperature: toNumberSafe(llm?.temperature),
|
||||
max_output_tokens: toNumberSafe(llm?.max_output_tokens)
|
||||
};
|
||||
}
|
||||
function textMojibakeScore(value) {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
|
||||
}
|
||||
function looksLikeMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
|
||||
}
|
||||
function repairAutogenMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibake(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
let improved = false;
|
||||
try {
|
||||
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScore(fromLatin1) > textMojibakeScore(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore
|
||||
}
|
||||
if (!improved) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
function sanitizeGeneratedQuestion(value) {
|
||||
return repairAutogenMojibake(String(value ?? ""))
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
function splitQuestionCandidates(rawText) {
|
||||
const normalized = repairAutogenMojibake(rawText).replace(/\r/g, "\n").trim();
|
||||
if (!normalized)
|
||||
return [];
|
||||
const unescaped = normalized.replace(/\\"/g, '"').replace(/\\n/g, "\n");
|
||||
const byLines = unescaped
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, ""))
|
||||
.map((line) => sanitizeGeneratedQuestion(line))
|
||||
.filter((line) => line.length > 0);
|
||||
if (byLines.length > 1) {
|
||||
return byLines;
|
||||
}
|
||||
const questionMarkCount = (unescaped.match(/\?/g) ?? []).length;
|
||||
if (questionMarkCount > 1) {
|
||||
const byQuestion = unescaped
|
||||
.split("?")
|
||||
.map((chunk) => sanitizeGeneratedQuestion(chunk))
|
||||
.filter((chunk) => chunk.length > 0)
|
||||
.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`));
|
||||
if (byQuestion.length > 1) {
|
||||
return byQuestion;
|
||||
}
|
||||
}
|
||||
const quoted = Array.from(unescaped.matchAll(/"([^"\n]{6,}?)"/g))
|
||||
.map((match) => sanitizeGeneratedQuestion(match[1]))
|
||||
.filter((line) => line.length > 0);
|
||||
if (quoted.length > 1) {
|
||||
return quoted;
|
||||
}
|
||||
const cleaned = sanitizeGeneratedQuestion(unescaped);
|
||||
return cleaned ? [cleaned] : [];
|
||||
}
|
||||
function parseAutogenOutputJson(rawText) {
|
||||
const cleaned = repairAutogenMojibake(rawText)
|
||||
.trim()
|
||||
.replace(/^```json\s*/i, "")
|
||||
.replace(/^```\s*/i, "")
|
||||
.replace(/```$/i, "")
|
||||
.trim();
|
||||
if (!cleaned)
|
||||
return null;
|
||||
try {
|
||||
return JSON.parse(cleaned);
|
||||
}
|
||||
catch {
|
||||
// continue
|
||||
}
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const arrayEnd = cleaned.lastIndexOf("]");
|
||||
if (arrayStart >= 0 && arrayEnd > arrayStart) {
|
||||
const fragment = cleaned.slice(arrayStart, arrayEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment);
|
||||
}
|
||||
catch {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
const objStart = cleaned.indexOf("{");
|
||||
const objEnd = cleaned.lastIndexOf("}");
|
||||
if (objStart >= 0 && objEnd > objStart) {
|
||||
const fragment = cleaned.slice(objStart, objEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment);
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function collectQuestionsFromCandidate(value, depth = 0) {
|
||||
if (depth > 5 || value === null || value === undefined) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const text = value.trim();
|
||||
if (!text)
|
||||
return [];
|
||||
const nestedParsed = parseAutogenOutputJson(text);
|
||||
if (nestedParsed !== null) {
|
||||
const nestedQuestions = collectQuestionsFromCandidate(nestedParsed, depth + 1);
|
||||
if (nestedQuestions.length > 0) {
|
||||
return nestedQuestions;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const decoded = JSON.parse(text);
|
||||
if (decoded !== text) {
|
||||
const decodedQuestions = collectQuestionsFromCandidate(decoded, depth + 1);
|
||||
if (decodedQuestions.length > 0) {
|
||||
return decodedQuestions;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore non-JSON strings
|
||||
}
|
||||
return splitQuestionCandidates(text);
|
||||
}
|
||||
const record = toRecord(value);
|
||||
if (!record) {
|
||||
return [];
|
||||
}
|
||||
const fromQuestions = collectQuestionsFromCandidate(record.questions, depth + 1);
|
||||
if (fromQuestions.length > 0) {
|
||||
return fromQuestions;
|
||||
}
|
||||
const fallbackText = toStringSafe(record.question ?? record.user_message ?? record.text);
|
||||
return fallbackText ? splitQuestionCandidates(fallbackText) : [];
|
||||
}
|
||||
function extractQuestionsFromAutogenOutput(rawText) {
|
||||
const parsed = parseAutogenOutputJson(rawText);
|
||||
const fromParsed = collectQuestionsFromCandidate(parsed);
|
||||
if (fromParsed.length > 0) {
|
||||
return fromParsed;
|
||||
}
|
||||
return collectQuestionsFromCandidate(rawText);
|
||||
}
|
||||
async function generateQwenSeedQuestionsLive(input) {
|
||||
const seedExamples = collectCanonicalQuestions(40);
|
||||
const fallbackExamples = fallbackDomainTemplates(input.domain);
|
||||
const examples = (seedExamples.length > 0 ? seedExamples : fallbackExamples).slice(0, 8);
|
||||
const personalityPrompt = input.personalityPrompt ??
|
||||
"Генерируй реалистичные вопросы бухгалтера по 1С. Разговорный стиль допустим, но смысл должен быть четким.";
|
||||
const repairedPersonalityPrompt = repairAutogenMojibake(personalityPrompt);
|
||||
const maxOutputTokens = clampInt(input.llmConfig.max_output_tokens, 300, 3000, 1200);
|
||||
const temperature = input.llmConfig.temperature === null ? 0.5 : Math.max(0, Math.min(1.5, input.llmConfig.temperature));
|
||||
const systemPrompt = [
|
||||
"Ты генератор вопросов для автопрогонов бухгалтерского ассистента по 1С.",
|
||||
"Возвращай только JSON и никаких пояснений.",
|
||||
"Ассистент работает в read-only режиме: не проси действий изменения базы."
|
||||
].join(" ");
|
||||
const repairedSystemPrompt = repairAutogenMojibake(systemPrompt);
|
||||
const developerPrompt = [
|
||||
`Нужно сгенерировать ровно ${input.count} вопросов.`,
|
||||
"Формат ответа строго:",
|
||||
'{"questions":["вопрос 1","вопрос 2"]}',
|
||||
"Требования:",
|
||||
"1) каждый вопрос отдельный, без дубликатов;",
|
||||
"2) живой пользовательский язык;",
|
||||
"3) допустимы легкие разговорные сокращения;",
|
||||
"4) не выдавай мета-комментарии и не описывай правила."
|
||||
].join("\n");
|
||||
const repairedDeveloperPrompt = repairAutogenMojibake(developerPrompt);
|
||||
const userMessage = [
|
||||
`Домен: ${input.domain ?? "general"}.`,
|
||||
`Промпт личности: ${repairedPersonalityPrompt}`,
|
||||
"Примеры ориентиров по стилю и тематике:",
|
||||
...examples.map((item, index) => `${index + 1}. ${item}`)
|
||||
].join("\n");
|
||||
const repairedUserMessage = repairAutogenMojibake(userMessage);
|
||||
const response = await input.client.chat({
|
||||
llmProvider: input.llmConfig.llm_provider,
|
||||
apiKey: input.llmConfig.api_key,
|
||||
model: input.llmConfig.model,
|
||||
baseUrl: input.llmConfig.base_url ?? undefined,
|
||||
temperature,
|
||||
maxOutputTokens: maxOutputTokens
|
||||
}, {
|
||||
systemPrompt: repairedSystemPrompt,
|
||||
developerPrompt: repairedDeveloperPrompt,
|
||||
userMessage: repairedUserMessage,
|
||||
temperature,
|
||||
maxOutputTokens
|
||||
});
|
||||
const extracted = extractQuestionsFromAutogenOutput(response.outputText);
|
||||
const normalized = Array.from(new Set(extracted.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
if (normalized.length === 0) {
|
||||
throw new http_1.ApiError("AUTOGEN_LLM_EMPTY_OUTPUT", "Qwen не вернул пригодные вопросы для автогенерации.", 502, {
|
||||
model: input.llmConfig.model
|
||||
});
|
||||
}
|
||||
const fallback = generateQwenSeedQuestions(input.count, input.domain);
|
||||
return Array.from(new Set([...normalized, ...fallback])).slice(0, input.count);
|
||||
}
|
||||
function hasAnyRunFilterQuery(query) {
|
||||
return Boolean(toStringSafe(query.from) ??
|
||||
toStringSafe(query.to) ??
|
||||
@@ -996,7 +1348,8 @@ function buildAutogenCaseSetFileName(mode, generationId) {
|
||||
return `assistant_autogen_${mode}_${stamp}_${generationId}.json`;
|
||||
}
|
||||
function buildAutogenCaseSetPayload(input) {
|
||||
const cases = input.questions.map((question, index) => ({
|
||||
const normalizedQuestions = Array.from(new Set(input.questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
const cases = normalizedQuestions.map((question, index) => ({
|
||||
case_id: `AUTO-${String(index + 1).padStart(3, "0")}`,
|
||||
scenario_tag: `${input.mode}_${input.domain ?? "general"}`,
|
||||
question_type: "direct",
|
||||
@@ -1103,7 +1456,7 @@ function collectPostAnalysis(annotations, runMap, limitPerQueue) {
|
||||
].slice(0, 60)
|
||||
};
|
||||
}
|
||||
function buildAutoRunsRouter() {
|
||||
function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIResponsesClient()) {
|
||||
const router = (0, express_1.Router)();
|
||||
router.get("/api/autoruns/history", (req, res) => {
|
||||
const filters = parseFilters(req.query);
|
||||
@@ -1175,9 +1528,22 @@ function buildAutoRunsRouter() {
|
||||
if (!run) {
|
||||
throw new http_1.ApiError("AUTORUN_NOT_FOUND", `Run not found: ${runId}`, 404);
|
||||
}
|
||||
const annotations = readAnnotations();
|
||||
if (caseId === "__all__") {
|
||||
const dialog = buildRunAggregateDialog(run, annotations);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
run_id: runId,
|
||||
case_id: "__all__",
|
||||
...dialog,
|
||||
annotations: annotations
|
||||
.filter((item) => item.run_id === runId)
|
||||
.sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))
|
||||
});
|
||||
return;
|
||||
}
|
||||
const sessionDialog = loadSessionDialog(runId, caseId);
|
||||
const dialog = sessionDialog ?? buildFallbackDialog(run, caseId);
|
||||
const annotations = readAnnotations();
|
||||
const messages = withMessageAnnotations(runId, caseId, dialog.messages, annotations);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
@@ -1307,6 +1673,9 @@ function buildAutoRunsRouter() {
|
||||
if (targetRole !== "assistant") {
|
||||
throw new http_1.ApiError("AUTORUN_MESSAGE_NOT_ASSISTANT", "Only assistant answers can be annotated", 400);
|
||||
}
|
||||
const pairedUserQuestion = [...dialog.messages.slice(0, messageIndex)]
|
||||
.reverse()
|
||||
.find((item) => (toStringSafe(item.role) ?? "") === "user");
|
||||
const nowIso = new Date().toISOString();
|
||||
const annotations = readAnnotations();
|
||||
const key = annotationKey(runId, caseId, messageIndex);
|
||||
@@ -1322,6 +1691,9 @@ function buildAutoRunsRouter() {
|
||||
comment,
|
||||
manual_case_decision: manualCaseDecision,
|
||||
annotation_author: annotationAuthor,
|
||||
resolved: existing?.resolved ?? false,
|
||||
resolved_at: existing?.resolved_at ?? null,
|
||||
resolved_by: existing?.resolved_by ?? null,
|
||||
created_at: existing?.created_at ?? nowIso,
|
||||
updated_at: nowIso,
|
||||
context: {
|
||||
@@ -1331,7 +1703,9 @@ function buildAutoRunsRouter() {
|
||||
eval_target: run.eval_target,
|
||||
prompt_version: toStringSafe(run.report.prompt_version),
|
||||
domain: caseSummary.domain,
|
||||
query_class: caseSummary.query_class
|
||||
query_class: caseSummary.query_class,
|
||||
question_text: toStringSafe(pairedUserQuestion?.text),
|
||||
answer_text: toStringSafe(targetMessage.text)
|
||||
}
|
||||
};
|
||||
if (existingIndex >= 0) {
|
||||
@@ -1353,6 +1727,49 @@ function buildAutoRunsRouter() {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.patch("/api/autoruns/annotations/:annotation_id", (req, res, next) => {
|
||||
try {
|
||||
const annotationId = toStringSafe(req.params.annotation_id);
|
||||
if (!annotationId) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_ID", "annotation_id is required", 400);
|
||||
}
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_PATCH", "JSON body is required", 400);
|
||||
}
|
||||
const resolved = toBooleanSafe(body.resolved);
|
||||
if (resolved === null) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_PATCH", "resolved flag is required", 400);
|
||||
}
|
||||
const resolvedBy = parseAnnotationAuthor(body.resolved_by);
|
||||
const annotations = readAnnotations();
|
||||
const index = annotations.findIndex((item) => item.annotation_id === annotationId);
|
||||
if (index < 0) {
|
||||
throw new http_1.ApiError("ANNOTATION_NOT_FOUND", `Annotation not found: ${annotationId}`, 404);
|
||||
}
|
||||
const nowIso = new Date().toISOString();
|
||||
const current = annotations[index];
|
||||
const updated = {
|
||||
...current,
|
||||
resolved,
|
||||
resolved_at: resolved ? nowIso : null,
|
||||
resolved_by: resolved ? resolvedBy ?? current.resolved_by ?? null : null,
|
||||
updated_at: nowIso
|
||||
};
|
||||
annotations[index] = updated;
|
||||
writeAnnotations(annotations);
|
||||
const statsByCase = buildAnnotationStatsMap(updated.run_id, annotations);
|
||||
const caseStats = statsByCase.get(updated.case_id) ?? null;
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
annotation: updated,
|
||||
case_annotation_stats: caseStats
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.get("/api/autoruns/manual-decision-schema", (_req, res) => {
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
@@ -1416,7 +1833,19 @@ function buildAutoRunsRouter() {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/autoruns/autogen/generate", (req, res, next) => {
|
||||
router.get("/api/autoruns/autogen/personality-catalog", (_req, res, next) => {
|
||||
try {
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
generated_at: new Date().toISOString(),
|
||||
items: buildAutogenPersonalityCatalog()
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
|
||||
try {
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
@@ -1428,9 +1857,25 @@ function buildAutoRunsRouter() {
|
||||
const persistCaseSet = toBooleanSafe(body.persist_to_eval_cases) ?? true;
|
||||
const generatedBy = parseAnnotationAuthor(body.generated_by);
|
||||
const context = toRecord(body.context);
|
||||
const questions = mode === "qwen_seed"
|
||||
? generateQwenSeedQuestions(count, domain)
|
||||
: generateCodexCreativeQuestions(count, domain);
|
||||
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
|
||||
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
|
||||
let questions = [];
|
||||
if (mode === "qwen_seed") {
|
||||
if (!llmConfig) {
|
||||
throw new http_1.ApiError("AUTOGEN_LLM_CONFIG_REQUIRED", "Для режима qwen_seed нужен активный LLM-контур (provider/model/baseUrl) из настроек подключения.", 400);
|
||||
}
|
||||
questions = await generateQwenSeedQuestionsLive({
|
||||
count,
|
||||
domain,
|
||||
personalityPrompt,
|
||||
llmConfig,
|
||||
client: openaiClient
|
||||
});
|
||||
}
|
||||
else {
|
||||
questions = generateCodexCreativeQuestions(count, domain);
|
||||
}
|
||||
questions = Array.from(new Set(questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))).slice(0, count);
|
||||
const generationId = generateAutogenId();
|
||||
let savedCaseSetFile = null;
|
||||
if (persistCaseSet) {
|
||||
@@ -1464,6 +1909,12 @@ function buildAutoRunsRouter() {
|
||||
assistant_prompt_version: toStringSafe(context.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(context.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(context.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(context.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(context.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(context.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
};
|
||||
|
||||
+363
-14
@@ -1,27 +1,275 @@
|
||||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.buildEvalRouter = buildEvalRouter;
|
||||
const fs_1 = __importDefault(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const nanoid_1 = require("nanoid");
|
||||
const express_1 = require("express");
|
||||
const config_1 = require("../config");
|
||||
const http_1 = require("../utils/http");
|
||||
const ASYNC_JOBS = new Map();
|
||||
const MAX_ASYNC_JOBS = 80;
|
||||
function toRecord(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function toStringSafe(value) {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
function toArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
function normalizeQuestionChunk(value) {
|
||||
return String(value ?? "")
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
function splitQuestionCandidate(raw) {
|
||||
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
const byLines = normalized
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, "").trim())
|
||||
.filter((line) => line.length > 0);
|
||||
const source = byLines.length > 1 ? byLines : [normalized];
|
||||
const chunks = [];
|
||||
for (const line of source) {
|
||||
const questionLike = Array.from(line.matchAll(/[^?]+(?:\?|$)/g))
|
||||
.map((match) => normalizeQuestionChunk(match[0]))
|
||||
.filter((item) => item.length > 0);
|
||||
if (questionLike.length > 1) {
|
||||
for (const item of questionLike) {
|
||||
chunks.push(item.endsWith("?") ? item : `${item}?`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chunks.push(normalizeQuestionChunk(line));
|
||||
}
|
||||
return chunks.filter((item) => item.length > 0);
|
||||
}
|
||||
function normalizeRuntimeQuestions(value) {
|
||||
const raw = toArray(value)
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
if (raw.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const expanded = raw.flatMap((item) => splitQuestionCandidate(item));
|
||||
const deduped = [];
|
||||
const seen = new Set();
|
||||
for (const item of expanded) {
|
||||
const normalized = normalizeQuestionChunk(item);
|
||||
if (!normalized)
|
||||
continue;
|
||||
if (seen.has(normalized))
|
||||
continue;
|
||||
seen.add(normalized);
|
||||
deduped.push(normalized);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
function normalizeCaseIds(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
function buildEvalPayloadFromBody(body) {
|
||||
return {
|
||||
normalizeConfig: (body.normalizeConfig ?? {}),
|
||||
caseIds: normalizeCaseIds(body.caseIds),
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: body.mode ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: body.eval_target ?? "normalizer",
|
||||
compareWithReportFile: typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
function resolveReadablePath(inputPath) {
|
||||
if (path_1.default.isAbsolute(inputPath)) {
|
||||
return inputPath;
|
||||
}
|
||||
const candidates = [
|
||||
path_1.default.resolve(config_1.EVAL_CASES_DIR, inputPath),
|
||||
path_1.default.resolve(config_1.EVAL_DATASETS_DIR, inputPath),
|
||||
path_1.default.resolve(inputPath)
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs_1.default.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
function readAssistantSuiteCaseSeeds(inputPath) {
|
||||
const filePath = resolveReadablePath(inputPath);
|
||||
const raw = fs_1.default.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
|
||||
const parsed = JSON.parse(raw);
|
||||
const record = toRecord(parsed);
|
||||
const cases = toArray(record?.cases);
|
||||
return cases
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item) => item !== null)
|
||||
.map((item) => {
|
||||
const caseId = toStringSafe(item.case_id);
|
||||
const turns = toArray(item.turns);
|
||||
if (!caseId || turns.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
case_id: caseId,
|
||||
turns_total: turns.length
|
||||
};
|
||||
})
|
||||
.filter((item) => item !== null);
|
||||
}
|
||||
function writeRuntimeAssistantSuiteFromQuestions(jobId, questions) {
|
||||
if (!fs_1.default.existsSync(config_1.EVAL_CASES_DIR)) {
|
||||
fs_1.default.mkdirSync(config_1.EVAL_CASES_DIR, { recursive: true });
|
||||
}
|
||||
const cases = questions.map((question, index) => {
|
||||
const caseId = `AUTO-${String(index + 1).padStart(3, "0")}`;
|
||||
return {
|
||||
case_id: caseId,
|
||||
scenario_tag: "autogen_runtime",
|
||||
question_type: "direct",
|
||||
broadness_level: "medium",
|
||||
turns: [{ user_message: question }]
|
||||
};
|
||||
});
|
||||
const payload = {
|
||||
suite_id: `assistant_autogen_runtime_${jobId}`,
|
||||
suite_version: "0.1.0",
|
||||
schema_version: "assistant_autogen_runtime_v0_1",
|
||||
scenario_count: cases.length,
|
||||
case_ids: cases.map((item) => item.case_id),
|
||||
cases
|
||||
};
|
||||
const fileName = `assistant_autogen_runtime_${jobId}.json`;
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
|
||||
return fileName;
|
||||
}
|
||||
function readSessionConversation(runId, caseId) {
|
||||
const sessionId = `${runId}-${caseId}`;
|
||||
const filePath = path_1.default.resolve(config_1.ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
|
||||
if (!fs_1.default.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(fs_1.default.readFileSync(filePath, "utf-8"));
|
||||
const record = toRecord(parsed);
|
||||
const conversation = toArray(record?.conversation)
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item) => item !== null);
|
||||
return conversation.map((item, index) => ({
|
||||
message_id: toStringSafe(item.message_id),
|
||||
role: toStringSafe(item.role) ?? "unknown",
|
||||
text: toStringSafe(item.text) ?? "",
|
||||
created_at: toStringSafe(item.created_at),
|
||||
trace_id: toStringSafe(item.trace_id),
|
||||
reply_type: toStringSafe(item.reply_type),
|
||||
message_index: index,
|
||||
case_id: caseId,
|
||||
case_message_index: index
|
||||
}));
|
||||
}
|
||||
catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
function syncJobWithSessions(job) {
|
||||
if (!job.run_id || !job.eval_target.startsWith("assistant_")) {
|
||||
return;
|
||||
}
|
||||
let completed = 0;
|
||||
let hasRunning = false;
|
||||
for (const item of job.cases) {
|
||||
const messages = readSessionConversation(job.run_id, item.case_id);
|
||||
item.messages = messages;
|
||||
const assistantMessages = messages.filter((entry) => entry.role === "assistant").length;
|
||||
const userMessages = messages.filter((entry) => entry.role === "user").length;
|
||||
if (assistantMessages >= item.turns_total && item.turns_total > 0) {
|
||||
item.status = "completed";
|
||||
completed += 1;
|
||||
continue;
|
||||
}
|
||||
if (userMessages > 0 || messages.length > 0) {
|
||||
item.status = "running";
|
||||
hasRunning = true;
|
||||
continue;
|
||||
}
|
||||
item.status = "queued";
|
||||
}
|
||||
job.completed_cases = completed;
|
||||
if (job.status === "running" && !hasRunning && completed === job.total_cases && job.total_cases > 0) {
|
||||
job.status = "completed";
|
||||
}
|
||||
}
|
||||
function trimAsyncJobsStore() {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS)
|
||||
return;
|
||||
const sorted = Array.from(ASYNC_JOBS.values()).sort((a, b) => Date.parse(a.updated_at) - Date.parse(b.updated_at));
|
||||
for (const item of sorted) {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS)
|
||||
break;
|
||||
ASYNC_JOBS.delete(item.job_id);
|
||||
}
|
||||
}
|
||||
function snapshotJob(job) {
|
||||
return {
|
||||
job_id: job.job_id,
|
||||
status: job.status,
|
||||
created_at: job.created_at,
|
||||
updated_at: job.updated_at,
|
||||
eval_target: job.eval_target,
|
||||
run_id: job.run_id,
|
||||
case_set_file: job.case_set_file,
|
||||
total_cases: job.total_cases,
|
||||
completed_cases: job.completed_cases,
|
||||
error: job.error,
|
||||
cases: job.cases,
|
||||
report_summary: job.report
|
||||
? {
|
||||
run_id: toStringSafe(job.report.run_id),
|
||||
run_timestamp: toStringSafe(job.report.run_timestamp) ?? toStringSafe(job.report.timestamp),
|
||||
score_index: typeof job.report.score_index === "number"
|
||||
? Number(job.report.score_index)
|
||||
: toRecord(job.report.metrics) && typeof toRecord(job.report.metrics)?.score_index === "number"
|
||||
? Number(toRecord(job.report.metrics)?.score_index)
|
||||
: null,
|
||||
cases_total: typeof job.report.cases_total === "number" ? Number(job.report.cases_total) : null
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
function buildEvalRouter(services) {
|
||||
const router = (0, express_1.Router)();
|
||||
router.post("/api/eval/run", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {});
|
||||
const report = await services.evalService.run({
|
||||
normalizeConfig: (body.normalizeConfig ?? {}),
|
||||
caseIds: Array.isArray(body.caseIds) ? body.caseIds : undefined,
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: body.mode ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: body.eval_target ?? "normalizer",
|
||||
compareWithReportFile: typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
const report = await services.evalService.run(payload);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
report
|
||||
@@ -31,5 +279,106 @@ function buildEvalRouter(services) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/eval/run-async/start", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
if (payload.evalTarget !== "assistant_stage1") {
|
||||
throw new http_1.ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
|
||||
}
|
||||
const questions = normalizeRuntimeQuestions(body.questions);
|
||||
const jobId = `job-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runtimeCaseSetFile = questions.length > 0
|
||||
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
|
||||
: payload.caseSetFile
|
||||
? payload.caseSetFile
|
||||
: undefined;
|
||||
if (!runtimeCaseSetFile) {
|
||||
throw new http_1.ApiError("ASYNC_CASESET_REQUIRED", "Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.", 400);
|
||||
}
|
||||
const caseSeeds = readAssistantSuiteCaseSeeds(runtimeCaseSetFile);
|
||||
if (caseSeeds.length === 0) {
|
||||
throw new http_1.ApiError("ASYNC_CASESET_EMPTY", "No runnable cases found in selected case-set.", 400);
|
||||
}
|
||||
const nowIso = new Date().toISOString();
|
||||
const job = {
|
||||
job_id: jobId,
|
||||
status: "queued",
|
||||
created_at: nowIso,
|
||||
updated_at: nowIso,
|
||||
eval_target: payload.evalTarget,
|
||||
run_id: runId,
|
||||
case_set_file: runtimeCaseSetFile,
|
||||
total_cases: caseSeeds.length,
|
||||
completed_cases: 0,
|
||||
cases: caseSeeds.map((item) => ({
|
||||
case_id: item.case_id,
|
||||
turns_total: item.turns_total,
|
||||
status: "queued",
|
||||
messages: []
|
||||
})),
|
||||
error: null,
|
||||
report: null
|
||||
};
|
||||
ASYNC_JOBS.set(job.job_id, job);
|
||||
trimAsyncJobsStore();
|
||||
setImmediate(() => {
|
||||
void (async () => {
|
||||
const target = ASYNC_JOBS.get(job.job_id);
|
||||
if (!target)
|
||||
return;
|
||||
target.status = "running";
|
||||
target.updated_at = new Date().toISOString();
|
||||
try {
|
||||
const report = await services.evalService.run({
|
||||
...payload,
|
||||
caseSetFile: runtimeCaseSetFile,
|
||||
runId
|
||||
});
|
||||
target.report = report;
|
||||
syncJobWithSessions(target);
|
||||
target.completed_cases = target.total_cases;
|
||||
target.status = "completed";
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
catch (error) {
|
||||
syncJobWithSessions(target);
|
||||
target.status = "failed";
|
||||
target.error = error instanceof Error ? error.message : String(error);
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
})();
|
||||
});
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.get("/api/eval/run-async/:job_id", (req, res, next) => {
|
||||
try {
|
||||
const jobId = String(req.params.job_id ?? "").trim();
|
||||
if (!jobId) {
|
||||
throw new http_1.ApiError("INVALID_ASYNC_JOB_ID", "job_id is required.", 400);
|
||||
}
|
||||
const job = ASYNC_JOBS.get(jobId);
|
||||
if (!job) {
|
||||
throw new http_1.ApiError("ASYNC_JOB_NOT_FOUND", `Async eval job not found: ${jobId}`, 404);
|
||||
}
|
||||
syncJobWithSessions(job);
|
||||
job.updated_at = new Date().toISOString();
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
return router;
|
||||
}
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ function createApp() {
|
||||
app.use((0, normalize_1.buildNormalizeRouter)(services));
|
||||
app.use((0, eval_1.buildEvalRouter)(services));
|
||||
app.use((0, assistant_1.buildAssistantRouter)(services));
|
||||
app.use((0, autoRuns_1.buildAutoRunsRouter)());
|
||||
app.use((0, autoRuns_1.buildAutoRunsRouter)(openaiClient));
|
||||
app.use((0, history_1.buildHistoryRouter)());
|
||||
app.use((0, presets_1.buildPresetsRouter)());
|
||||
app.use((0, accountingAgent_1.buildAccountingAgentRouter)(services));
|
||||
|
||||
@@ -4834,6 +4834,10 @@ class AssistantService {
|
||||
debug: null
|
||||
};
|
||||
this.sessions.appendItem(sessionId, userItem);
|
||||
const sessionAfterUserAppend = this.sessions.getSession(sessionId);
|
||||
if (sessionAfterUserAppend) {
|
||||
this.sessionLogger.persistSession(sessionAfterUserAppend);
|
||||
}
|
||||
const sessionOrganizationScope = resolveSessionOrganizationScopeContext(userMessage, session.items);
|
||||
const finalizeAddressLaneResponse = (addressLane, effectiveAddressUserMessage, carryoverMeta = null, llmPreDecomposeMeta = null) => {
|
||||
const safeAddressReply = sanitizeOutgoingAssistantText(addressLane.reply_text);
|
||||
|
||||
+8
-4
@@ -1552,7 +1552,7 @@ class EvalService {
|
||||
}
|
||||
const suite = parseAssistantSuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
|
||||
const diagnostics = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1568,6 +1568,7 @@ class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -1885,7 +1886,7 @@ class EvalService {
|
||||
}
|
||||
const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
|
||||
const diagnostics = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1903,6 +1904,7 @@ class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2177,7 +2179,8 @@ class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_stage2") {
|
||||
@@ -2187,7 +2190,8 @@ class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_p0") {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { Router } from "express";
|
||||
import iconv from "iconv-lite";
|
||||
import {
|
||||
ASSISTANT_SESSIONS_DIR,
|
||||
AUTORUN_ANNOTATIONS_FILE,
|
||||
@@ -11,7 +12,8 @@ import {
|
||||
REPORTS_DIR
|
||||
} from "../config";
|
||||
import { ApiError, ok } from "../utils/http";
|
||||
import { loadCapabilitiesRegistry, resolveNearestCapabilityGroup } from "../services/capabilitiesRegistry";
|
||||
import { loadCapabilitiesRegistry, resolveNearestCapabilityGroup, type CapabilityGroup } from "../services/capabilitiesRegistry";
|
||||
import { OpenAIResponsesClient } from "../services/openaiResponsesClient";
|
||||
|
||||
type AutoRunTarget = "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0" | "unknown";
|
||||
type AutoRunTrend = "up" | "down" | "flat";
|
||||
@@ -144,6 +146,9 @@ interface AutoRunAnnotationRecord {
|
||||
comment: string;
|
||||
manual_case_decision: ManualCaseDecision;
|
||||
annotation_author: string | null;
|
||||
resolved: boolean;
|
||||
resolved_at: string | null;
|
||||
resolved_by: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
context: {
|
||||
@@ -154,6 +159,8 @@ interface AutoRunAnnotationRecord {
|
||||
prompt_version: string | null;
|
||||
domain: string | null;
|
||||
query_class: string | null;
|
||||
question_text: string | null;
|
||||
answer_text: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,9 +185,28 @@ interface AutoGenHistoryRecord {
|
||||
assistant_prompt_version: string | null;
|
||||
decomposition_prompt_version: string | null;
|
||||
prompt_fingerprint: string | null;
|
||||
autogen_personality_id: string | null;
|
||||
autogen_personality_prompt: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
interface AutoGenPersonalityCatalogItem {
|
||||
id: string;
|
||||
label: string;
|
||||
domain: string | null;
|
||||
default_prompt: string;
|
||||
source: "built_in" | "capabilities_registry";
|
||||
}
|
||||
|
||||
interface AutoGenLlmRuntimeConfig {
|
||||
llm_provider: "openai" | "local";
|
||||
api_key: string;
|
||||
model: string;
|
||||
base_url: string | null;
|
||||
temperature: number | null;
|
||||
max_output_tokens: number | null;
|
||||
}
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
@@ -254,6 +280,11 @@ function parseAnnotationAuthor(value: unknown): string | null {
|
||||
return author.slice(0, 80);
|
||||
}
|
||||
|
||||
function parseAnnotationResolved(value: unknown, fallback = false): boolean {
|
||||
const parsed = toBooleanSafe(value);
|
||||
return parsed === null ? fallback : parsed;
|
||||
}
|
||||
|
||||
function readManualDecisionSchema(): Record<string, unknown> {
|
||||
const fallback: Record<string, unknown> = {
|
||||
schema_version: "manual_case_decision_schema_v1_fallback",
|
||||
@@ -300,6 +331,8 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
|
||||
questions: toArray(item.questions)
|
||||
.map((q) => toStringSafe(q))
|
||||
.filter((q): q is string => q !== null)
|
||||
.map((q) => sanitizeGeneratedQuestion(q))
|
||||
.filter((q) => q.length > 0)
|
||||
.slice(0, 500),
|
||||
generated_by: toStringSafe(item.generated_by),
|
||||
saved_case_set_file: toStringSafe(item.saved_case_set_file),
|
||||
@@ -310,6 +343,12 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
|
||||
assistant_prompt_version: toStringSafe(toRecord(item.context)?.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(toRecord(item.context)?.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(toRecord(item.context)?.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
}))
|
||||
@@ -356,11 +395,11 @@ function collectCanonicalQuestions(limit = 300): string[] {
|
||||
for (const testCase of cases) {
|
||||
const rawQuestion = toStringSafe(testCase.raw_question) ?? toStringSafe(testCase.user_message) ?? toStringSafe(testCase.query);
|
||||
if (rawQuestion) {
|
||||
questions.push(rawQuestion);
|
||||
questions.push(sanitizeGeneratedQuestion(rawQuestion));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(new Set(questions)).slice(0, limit);
|
||||
return Array.from(new Set(questions.filter((item) => item.length > 0))).slice(0, limit);
|
||||
}
|
||||
|
||||
function normalizeDomainHint(value: unknown): string | null {
|
||||
@@ -369,6 +408,55 @@ function normalizeDomainHint(value: unknown): string | null {
|
||||
return domain.toLowerCase();
|
||||
}
|
||||
|
||||
function buildAutogenPromptFromCapabilityGroup(group: CapabilityGroup): string {
|
||||
const supported = group.supported_operations.slice(0, 3).join(", ");
|
||||
const examples = group.typical_queries.slice(0, 2).join(" | ");
|
||||
const hints = group.one_c_hints.slice(0, 2).join(", ");
|
||||
const operationsPart = supported ? ` Опирайся на операции: ${supported}.` : "";
|
||||
const examplesPart = examples ? ` Ближайшие формулировки: ${examples}.` : "";
|
||||
const hintsPart = hints ? ` Можно мягко упоминать контекст 1С: ${hints}.` : "";
|
||||
return (
|
||||
`Генерируй реалистичные вопросы бухгалтера по группе "${group.group_title}".` +
|
||||
` Добавляй живую разговорную форму и опечатки, но сохраняй бизнес-смысл.${operationsPart}${examplesPart}${hintsPart}` +
|
||||
" Не выдумывай операции вне read-only режима."
|
||||
);
|
||||
}
|
||||
|
||||
function buildAutogenPersonalityCatalog(): AutoGenPersonalityCatalogItem[] {
|
||||
const builtIn: AutoGenPersonalityCatalogItem[] = [
|
||||
{
|
||||
id: "general",
|
||||
label: "Общий контур",
|
||||
domain: null,
|
||||
default_prompt:
|
||||
"Генерируй реалистичные живые вопросы бухгалтера по 1С. Добавляй разговорные формулировки и опечатки, но сохраняй бизнес-смысл.",
|
||||
source: "built_in"
|
||||
}
|
||||
];
|
||||
|
||||
const registry = loadCapabilitiesRegistry();
|
||||
const registryBased = registry.groups.map<AutoGenPersonalityCatalogItem>((group) => ({
|
||||
id: `registry_${group.group_code}`,
|
||||
label: `${group.group_title} (реестр)`,
|
||||
domain: group.group_code,
|
||||
default_prompt: buildAutogenPromptFromCapabilityGroup(group),
|
||||
source: "capabilities_registry"
|
||||
}));
|
||||
|
||||
const dedup = new Map<string, AutoGenPersonalityCatalogItem>();
|
||||
for (const item of [...builtIn, ...registryBased]) {
|
||||
if (!item.id.trim()) continue;
|
||||
if (!dedup.has(item.id)) {
|
||||
dedup.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return [...dedup.values()].map((item) => ({
|
||||
...item,
|
||||
label: repairAutogenMojibake(item.label),
|
||||
default_prompt: repairAutogenMojibake(item.default_prompt)
|
||||
}));
|
||||
}
|
||||
|
||||
function fallbackDomainTemplates(domain: string | null): string[] {
|
||||
if (domain?.includes("vat") || domain?.includes("ндс")) {
|
||||
return [
|
||||
@@ -428,9 +516,9 @@ function generateQwenSeedQuestions(count: number, domain: string | null): string
|
||||
const out: string[] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = bag[index % bag.length];
|
||||
out.push(mutateIntoQwenStyle(base, index));
|
||||
out.push(sanitizeGeneratedQuestion(mutateIntoQwenStyle(base, index)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
|
||||
function generateCodexCreativeQuestions(count: number, domain: string | null): string[] {
|
||||
@@ -446,9 +534,9 @@ function generateCodexCreativeQuestions(count: number, domain: string | null): s
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = domainTemplates[index % domainTemplates.length];
|
||||
const pattern = patterns[index % patterns.length];
|
||||
out.push(pattern.replace("{q}", base));
|
||||
out.push(sanitizeGeneratedQuestion(pattern.replace("{q}", base)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
|
||||
function generateAutogenId(): string {
|
||||
@@ -480,6 +568,9 @@ function readAnnotations(): AutoRunAnnotationRecord[] {
|
||||
comment: toStringSafe(item.comment) ?? "",
|
||||
manual_case_decision: parseManualCaseDecision(item.manual_case_decision),
|
||||
annotation_author: parseAnnotationAuthor(item.annotation_author),
|
||||
resolved: parseAnnotationResolved(item.resolved),
|
||||
resolved_at: toStringSafe(item.resolved_at),
|
||||
resolved_by: parseAnnotationAuthor(item.resolved_by),
|
||||
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
|
||||
updated_at: toStringSafe(item.updated_at) ?? new Date().toISOString(),
|
||||
context: {
|
||||
@@ -489,7 +580,9 @@ function readAnnotations(): AutoRunAnnotationRecord[] {
|
||||
eval_target: (toStringSafe(context?.eval_target) as AutoRunTarget | null) ?? "unknown",
|
||||
prompt_version: toStringSafe(context?.prompt_version),
|
||||
domain: toStringSafe(context?.domain),
|
||||
query_class: toStringSafe(context?.query_class)
|
||||
query_class: toStringSafe(context?.query_class),
|
||||
question_text: toStringSafe(context?.question_text),
|
||||
answer_text: toStringSafe(context?.answer_text)
|
||||
}
|
||||
} satisfies AutoRunAnnotationRecord;
|
||||
})
|
||||
@@ -1156,6 +1249,51 @@ function withMessageAnnotations(
|
||||
});
|
||||
}
|
||||
|
||||
function buildRunAggregateDialog(
|
||||
run: IndexedRun,
|
||||
annotations: AutoRunAnnotationRecord[]
|
||||
): {
|
||||
source: "run_aggregate";
|
||||
session_id: string;
|
||||
messages: Array<Record<string, unknown>>;
|
||||
decomposition: string[];
|
||||
assistant_mode: Record<string, unknown> | null;
|
||||
} {
|
||||
const cases = buildCaseSummaries(run.report, run.run_id, false);
|
||||
const messages: Array<Record<string, unknown>> = [];
|
||||
const decomposition: string[] = [];
|
||||
let globalMessageIndex = 0;
|
||||
|
||||
for (const item of cases) {
|
||||
const caseId = item.case_id;
|
||||
const caseDialog = loadSessionDialog(run.run_id, caseId) ?? buildFallbackDialog(run, caseId);
|
||||
const annotatedCaseMessages = withMessageAnnotations(run.run_id, caseId, caseDialog.messages, annotations);
|
||||
|
||||
for (const caseMessage of annotatedCaseMessages) {
|
||||
const localMessageIndex = toNumberSafe(caseMessage.message_index) ?? 0;
|
||||
messages.push({
|
||||
...caseMessage,
|
||||
case_id: caseId,
|
||||
case_message_index: localMessageIndex,
|
||||
message_index: globalMessageIndex
|
||||
});
|
||||
globalMessageIndex += 1;
|
||||
}
|
||||
|
||||
if (caseDialog.decomposition.length > 0) {
|
||||
decomposition.push(...caseDialog.decomposition.map((step) => `[${caseId}] ${step}`));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: "run_aggregate",
|
||||
session_id: `${run.run_id}::__all__`,
|
||||
messages,
|
||||
decomposition,
|
||||
assistant_mode: null
|
||||
};
|
||||
}
|
||||
|
||||
function generateAnnotationId(): string {
|
||||
return `ann-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
}
|
||||
@@ -1189,6 +1327,300 @@ function parseAutogenDomain(value: unknown): string | null {
|
||||
return domain.slice(0, 80);
|
||||
}
|
||||
|
||||
function parseAutogenLlmRuntimeConfig(
|
||||
body: Record<string, unknown>,
|
||||
context: Record<string, unknown> | null
|
||||
): AutoGenLlmRuntimeConfig | null {
|
||||
const llm = toRecord(body.llm);
|
||||
const providerRaw = toStringSafe(llm?.llm_provider ?? context?.llm_provider)?.toLowerCase() ?? "";
|
||||
const model = toStringSafe(llm?.model ?? context?.model);
|
||||
if (!model || (providerRaw !== "openai" && providerRaw !== "local")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
llm_provider: providerRaw === "local" ? "local" : "openai",
|
||||
api_key: toStringSafe(llm?.api_key) ?? "",
|
||||
model,
|
||||
base_url: toStringSafe(llm?.base_url),
|
||||
temperature: toNumberSafe(llm?.temperature),
|
||||
max_output_tokens: toNumberSafe(llm?.max_output_tokens)
|
||||
};
|
||||
}
|
||||
|
||||
function textMojibakeScore(value: string): number {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
|
||||
}
|
||||
|
||||
function looksLikeMojibake(value: string): boolean {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
|
||||
}
|
||||
|
||||
function repairAutogenMojibake(value: string): string {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibake(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
let improved = false;
|
||||
try {
|
||||
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
improved = true;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScore(fromLatin1) > textMojibakeScore(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
improved = true;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (!improved) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function sanitizeGeneratedQuestion(value: string): string {
|
||||
return repairAutogenMojibake(String(value ?? ""))
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitQuestionCandidates(rawText: string): string[] {
|
||||
const normalized = repairAutogenMojibake(rawText).replace(/\r/g, "\n").trim();
|
||||
if (!normalized) return [];
|
||||
|
||||
const unescaped = normalized.replace(/\\"/g, '"').replace(/\\n/g, "\n");
|
||||
const byLines = unescaped
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, ""))
|
||||
.map((line) => sanitizeGeneratedQuestion(line))
|
||||
.filter((line) => line.length > 0);
|
||||
if (byLines.length > 1) {
|
||||
return byLines;
|
||||
}
|
||||
|
||||
const questionMarkCount = (unescaped.match(/\?/g) ?? []).length;
|
||||
if (questionMarkCount > 1) {
|
||||
const byQuestion = unescaped
|
||||
.split("?")
|
||||
.map((chunk) => sanitizeGeneratedQuestion(chunk))
|
||||
.filter((chunk) => chunk.length > 0)
|
||||
.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`));
|
||||
if (byQuestion.length > 1) {
|
||||
return byQuestion;
|
||||
}
|
||||
}
|
||||
|
||||
const quoted = Array.from(unescaped.matchAll(/"([^"\n]{6,}?)"/g))
|
||||
.map((match) => sanitizeGeneratedQuestion(match[1]))
|
||||
.filter((line) => line.length > 0);
|
||||
if (quoted.length > 1) {
|
||||
return quoted;
|
||||
}
|
||||
|
||||
const cleaned = sanitizeGeneratedQuestion(unescaped);
|
||||
return cleaned ? [cleaned] : [];
|
||||
}
|
||||
|
||||
function parseAutogenOutputJson(rawText: string): unknown | null {
|
||||
const cleaned = repairAutogenMojibake(rawText)
|
||||
.trim()
|
||||
.replace(/^```json\s*/i, "")
|
||||
.replace(/^```\s*/i, "")
|
||||
.replace(/```$/i, "")
|
||||
.trim();
|
||||
if (!cleaned) return null;
|
||||
try {
|
||||
return JSON.parse(cleaned) as unknown;
|
||||
} catch {
|
||||
// continue
|
||||
}
|
||||
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const arrayEnd = cleaned.lastIndexOf("]");
|
||||
if (arrayStart >= 0 && arrayEnd > arrayStart) {
|
||||
const fragment = cleaned.slice(arrayStart, arrayEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment) as unknown;
|
||||
} catch {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
|
||||
const objStart = cleaned.indexOf("{");
|
||||
const objEnd = cleaned.lastIndexOf("}");
|
||||
if (objStart >= 0 && objEnd > objStart) {
|
||||
const fragment = cleaned.slice(objStart, objEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment) as unknown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectQuestionsFromCandidate(value: unknown, depth = 0): string[] {
|
||||
if (depth > 5 || value === null || value === undefined) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const text = value.trim();
|
||||
if (!text) return [];
|
||||
|
||||
const nestedParsed = parseAutogenOutputJson(text);
|
||||
if (nestedParsed !== null) {
|
||||
const nestedQuestions = collectQuestionsFromCandidate(nestedParsed, depth + 1);
|
||||
if (nestedQuestions.length > 0) {
|
||||
return nestedQuestions;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = JSON.parse(text) as unknown;
|
||||
if (decoded !== text) {
|
||||
const decodedQuestions = collectQuestionsFromCandidate(decoded, depth + 1);
|
||||
if (decodedQuestions.length > 0) {
|
||||
return decodedQuestions;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore non-JSON strings
|
||||
}
|
||||
|
||||
return splitQuestionCandidates(text);
|
||||
}
|
||||
|
||||
const record = toRecord(value);
|
||||
if (!record) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const fromQuestions = collectQuestionsFromCandidate(record.questions, depth + 1);
|
||||
if (fromQuestions.length > 0) {
|
||||
return fromQuestions;
|
||||
}
|
||||
|
||||
const fallbackText = toStringSafe(record.question ?? record.user_message ?? record.text);
|
||||
return fallbackText ? splitQuestionCandidates(fallbackText) : [];
|
||||
}
|
||||
|
||||
function extractQuestionsFromAutogenOutput(rawText: string): string[] {
|
||||
const parsed = parseAutogenOutputJson(rawText);
|
||||
const fromParsed = collectQuestionsFromCandidate(parsed);
|
||||
if (fromParsed.length > 0) {
|
||||
return fromParsed;
|
||||
}
|
||||
return collectQuestionsFromCandidate(rawText);
|
||||
}
|
||||
|
||||
async function generateQwenSeedQuestionsLive(input: {
|
||||
count: number;
|
||||
domain: string | null;
|
||||
personalityPrompt: string | null;
|
||||
llmConfig: AutoGenLlmRuntimeConfig;
|
||||
client: OpenAIResponsesClient;
|
||||
}): Promise<string[]> {
|
||||
const seedExamples = collectCanonicalQuestions(40);
|
||||
const fallbackExamples = fallbackDomainTemplates(input.domain);
|
||||
const examples = (seedExamples.length > 0 ? seedExamples : fallbackExamples).slice(0, 8);
|
||||
const personalityPrompt =
|
||||
input.personalityPrompt ??
|
||||
"Генерируй реалистичные вопросы бухгалтера по 1С. Разговорный стиль допустим, но смысл должен быть четким.";
|
||||
const repairedPersonalityPrompt = repairAutogenMojibake(personalityPrompt);
|
||||
const maxOutputTokens = clampInt(input.llmConfig.max_output_tokens, 300, 3000, 1200);
|
||||
const temperature = input.llmConfig.temperature === null ? 0.5 : Math.max(0, Math.min(1.5, input.llmConfig.temperature));
|
||||
|
||||
const systemPrompt = [
|
||||
"Ты генератор вопросов для автопрогонов бухгалтерского ассистента по 1С.",
|
||||
"Возвращай только JSON и никаких пояснений.",
|
||||
"Ассистент работает в read-only режиме: не проси действий изменения базы."
|
||||
].join(" ");
|
||||
const repairedSystemPrompt = repairAutogenMojibake(systemPrompt);
|
||||
|
||||
const developerPrompt = [
|
||||
`Нужно сгенерировать ровно ${input.count} вопросов.`,
|
||||
"Формат ответа строго:",
|
||||
'{"questions":["вопрос 1","вопрос 2"]}',
|
||||
"Требования:",
|
||||
"1) каждый вопрос отдельный, без дубликатов;",
|
||||
"2) живой пользовательский язык;",
|
||||
"3) допустимы легкие разговорные сокращения;",
|
||||
"4) не выдавай мета-комментарии и не описывай правила."
|
||||
].join("\n");
|
||||
const repairedDeveloperPrompt = repairAutogenMojibake(developerPrompt);
|
||||
|
||||
const userMessage = [
|
||||
`Домен: ${input.domain ?? "general"}.`,
|
||||
`Промпт личности: ${repairedPersonalityPrompt}`,
|
||||
"Примеры ориентиров по стилю и тематике:",
|
||||
...examples.map((item, index) => `${index + 1}. ${item}`)
|
||||
].join("\n");
|
||||
const repairedUserMessage = repairAutogenMojibake(userMessage);
|
||||
|
||||
const response = await input.client.chat(
|
||||
{
|
||||
llmProvider: input.llmConfig.llm_provider,
|
||||
apiKey: input.llmConfig.api_key,
|
||||
model: input.llmConfig.model,
|
||||
baseUrl: input.llmConfig.base_url ?? undefined,
|
||||
temperature,
|
||||
maxOutputTokens: maxOutputTokens
|
||||
},
|
||||
{
|
||||
systemPrompt: repairedSystemPrompt,
|
||||
developerPrompt: repairedDeveloperPrompt,
|
||||
userMessage: repairedUserMessage,
|
||||
temperature,
|
||||
maxOutputTokens
|
||||
}
|
||||
);
|
||||
|
||||
const extracted = extractQuestionsFromAutogenOutput(response.outputText);
|
||||
const normalized = Array.from(new Set(extracted.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
if (normalized.length === 0) {
|
||||
throw new ApiError("AUTOGEN_LLM_EMPTY_OUTPUT", "Qwen не вернул пригодные вопросы для автогенерации.", 502, {
|
||||
model: input.llmConfig.model
|
||||
});
|
||||
}
|
||||
|
||||
const fallback = generateQwenSeedQuestions(input.count, input.domain);
|
||||
return Array.from(new Set([...normalized, ...fallback])).slice(0, input.count);
|
||||
}
|
||||
|
||||
function hasAnyRunFilterQuery(query: Record<string, unknown>): boolean {
|
||||
return Boolean(
|
||||
toStringSafe(query.from) ??
|
||||
@@ -1219,7 +1651,10 @@ function buildAutogenCaseSetPayload(input: {
|
||||
domain: string | null;
|
||||
questions: string[];
|
||||
}): Record<string, unknown> {
|
||||
const cases = input.questions.map((question, index) => ({
|
||||
const normalizedQuestions = Array.from(
|
||||
new Set(input.questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))
|
||||
);
|
||||
const cases = normalizedQuestions.map((question, index) => ({
|
||||
case_id: `AUTO-${String(index + 1).padStart(3, "0")}`,
|
||||
scenario_tag: `${input.mode}_${input.domain ?? "general"}`,
|
||||
question_type: "direct",
|
||||
@@ -1341,7 +1776,7 @@ function collectPostAnalysis(
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAutoRunsRouter(): Router {
|
||||
export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()): Router {
|
||||
const router = Router();
|
||||
|
||||
router.get("/api/autoruns/history", (req, res) => {
|
||||
@@ -1423,9 +1858,23 @@ export function buildAutoRunsRouter(): Router {
|
||||
throw new ApiError("AUTORUN_NOT_FOUND", `Run not found: ${runId}`, 404);
|
||||
}
|
||||
|
||||
const annotations = readAnnotations();
|
||||
if (caseId === "__all__") {
|
||||
const dialog = buildRunAggregateDialog(run, annotations);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
run_id: runId,
|
||||
case_id: "__all__",
|
||||
...dialog,
|
||||
annotations: annotations
|
||||
.filter((item) => item.run_id === runId)
|
||||
.sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionDialog = loadSessionDialog(runId, caseId);
|
||||
const dialog = sessionDialog ?? buildFallbackDialog(run, caseId);
|
||||
const annotations = readAnnotations();
|
||||
const messages = withMessageAnnotations(runId, caseId, dialog.messages, annotations);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
@@ -1563,6 +2012,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
if (targetRole !== "assistant") {
|
||||
throw new ApiError("AUTORUN_MESSAGE_NOT_ASSISTANT", "Only assistant answers can be annotated", 400);
|
||||
}
|
||||
const pairedUserQuestion = [...dialog.messages.slice(0, messageIndex)]
|
||||
.reverse()
|
||||
.find((item) => (toStringSafe(item.role) ?? "") === "user");
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const annotations = readAnnotations();
|
||||
@@ -1580,6 +2032,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
comment,
|
||||
manual_case_decision: manualCaseDecision,
|
||||
annotation_author: annotationAuthor,
|
||||
resolved: existing?.resolved ?? false,
|
||||
resolved_at: existing?.resolved_at ?? null,
|
||||
resolved_by: existing?.resolved_by ?? null,
|
||||
created_at: existing?.created_at ?? nowIso,
|
||||
updated_at: nowIso,
|
||||
context: {
|
||||
@@ -1589,7 +2044,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
eval_target: run.eval_target,
|
||||
prompt_version: toStringSafe(run.report.prompt_version),
|
||||
domain: caseSummary.domain,
|
||||
query_class: caseSummary.query_class
|
||||
query_class: caseSummary.query_class,
|
||||
question_text: toStringSafe(pairedUserQuestion?.text),
|
||||
answer_text: toStringSafe(targetMessage.text)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1613,6 +2070,56 @@ export function buildAutoRunsRouter(): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.patch("/api/autoruns/annotations/:annotation_id", (req, res, next) => {
|
||||
try {
|
||||
const annotationId = toStringSafe(req.params.annotation_id);
|
||||
if (!annotationId) {
|
||||
throw new ApiError("INVALID_ANNOTATION_ID", "annotation_id is required", 400);
|
||||
}
|
||||
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
throw new ApiError("INVALID_ANNOTATION_PATCH", "JSON body is required", 400);
|
||||
}
|
||||
|
||||
const resolved = toBooleanSafe(body.resolved);
|
||||
if (resolved === null) {
|
||||
throw new ApiError("INVALID_ANNOTATION_PATCH", "resolved flag is required", 400);
|
||||
}
|
||||
const resolvedBy = parseAnnotationAuthor(body.resolved_by);
|
||||
|
||||
const annotations = readAnnotations();
|
||||
const index = annotations.findIndex((item) => item.annotation_id === annotationId);
|
||||
if (index < 0) {
|
||||
throw new ApiError("ANNOTATION_NOT_FOUND", `Annotation not found: ${annotationId}`, 404);
|
||||
}
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const current = annotations[index];
|
||||
const updated: AutoRunAnnotationRecord = {
|
||||
...current,
|
||||
resolved,
|
||||
resolved_at: resolved ? nowIso : null,
|
||||
resolved_by: resolved ? resolvedBy ?? current.resolved_by ?? null : null,
|
||||
updated_at: nowIso
|
||||
};
|
||||
|
||||
annotations[index] = updated;
|
||||
writeAnnotations(annotations);
|
||||
|
||||
const statsByCase = buildAnnotationStatsMap(updated.run_id, annotations);
|
||||
const caseStats = statsByCase.get(updated.case_id) ?? null;
|
||||
|
||||
ok(res, {
|
||||
ok: true,
|
||||
annotation: updated,
|
||||
case_annotation_stats: caseStats
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/api/autoruns/manual-decision-schema", (_req, res) => {
|
||||
ok(res, {
|
||||
ok: true,
|
||||
@@ -1682,7 +2189,19 @@ export function buildAutoRunsRouter(): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/autoruns/autogen/generate", (req, res, next) => {
|
||||
router.get("/api/autoruns/autogen/personality-catalog", (_req, res, next) => {
|
||||
try {
|
||||
ok(res, {
|
||||
ok: true,
|
||||
generated_at: new Date().toISOString(),
|
||||
items: buildAutogenPersonalityCatalog()
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
|
||||
try {
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
@@ -1694,11 +2213,32 @@ export function buildAutoRunsRouter(): Router {
|
||||
const persistCaseSet = toBooleanSafe(body.persist_to_eval_cases) ?? true;
|
||||
const generatedBy = parseAnnotationAuthor(body.generated_by);
|
||||
const context = toRecord(body.context);
|
||||
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
|
||||
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
|
||||
|
||||
const questions =
|
||||
mode === "qwen_seed"
|
||||
? generateQwenSeedQuestions(count, domain)
|
||||
: generateCodexCreativeQuestions(count, domain);
|
||||
let questions: string[] = [];
|
||||
if (mode === "qwen_seed") {
|
||||
if (!llmConfig) {
|
||||
throw new ApiError(
|
||||
"AUTOGEN_LLM_CONFIG_REQUIRED",
|
||||
"Для режима qwen_seed нужен активный LLM-контур (provider/model/baseUrl) из настроек подключения.",
|
||||
400
|
||||
);
|
||||
}
|
||||
questions = await generateQwenSeedQuestionsLive({
|
||||
count,
|
||||
domain,
|
||||
personalityPrompt,
|
||||
llmConfig,
|
||||
client: openaiClient
|
||||
});
|
||||
} else {
|
||||
questions = generateCodexCreativeQuestions(count, domain);
|
||||
}
|
||||
questions = Array.from(new Set(questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))).slice(
|
||||
0,
|
||||
count
|
||||
);
|
||||
const generationId = generateAutogenId();
|
||||
|
||||
let savedCaseSetFile: string | null = null;
|
||||
@@ -1734,6 +2274,12 @@ export function buildAutoRunsRouter(): Router {
|
||||
assistant_prompt_version: toStringSafe(context.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(context.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(context.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(context.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(context.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(context.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
};
|
||||
@@ -1752,3 +2298,5 @@ export function buildAutoRunsRouter(): Router {
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,30 +1,333 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { nanoid } from "nanoid";
|
||||
import { Router } from "express";
|
||||
import { ASSISTANT_SESSIONS_DIR, EVAL_CASES_DIR, EVAL_DATASETS_DIR } from "../config";
|
||||
import type { AppServices } from "../serverContext";
|
||||
import { ok } from "../utils/http";
|
||||
import { ApiError, ok } from "../utils/http";
|
||||
import type { EvalRunMode, NormalizeRequestPayload } from "../types/normalizer";
|
||||
import type { EvalTarget } from "../types/assistantEval";
|
||||
|
||||
type EvalAsyncStatus = "queued" | "running" | "completed" | "failed";
|
||||
|
||||
interface EvalAsyncCaseInfo {
|
||||
case_id: string;
|
||||
turns_total: number;
|
||||
status: EvalAsyncStatus;
|
||||
messages: Array<{
|
||||
message_id: string | null;
|
||||
role: string;
|
||||
text: string;
|
||||
created_at: string | null;
|
||||
trace_id: string | null;
|
||||
reply_type: string | null;
|
||||
message_index: number;
|
||||
case_id: string;
|
||||
case_message_index: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface EvalAsyncJob {
|
||||
job_id: string;
|
||||
status: EvalAsyncStatus;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
eval_target: EvalTarget;
|
||||
run_id: string;
|
||||
case_set_file: string | null;
|
||||
total_cases: number;
|
||||
completed_cases: number;
|
||||
cases: EvalAsyncCaseInfo[];
|
||||
error: string | null;
|
||||
report: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
const ASYNC_JOBS = new Map<string, EvalAsyncJob>();
|
||||
const MAX_ASYNC_JOBS = 80;
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function toStringSafe(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function toArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function normalizeQuestionChunk(value: string): string {
|
||||
return String(value ?? "")
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitQuestionCandidate(raw: string): string[] {
|
||||
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const byLines = normalized
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, "").trim())
|
||||
.filter((line) => line.length > 0);
|
||||
const source = byLines.length > 1 ? byLines : [normalized];
|
||||
|
||||
const chunks: string[] = [];
|
||||
for (const line of source) {
|
||||
const questionLike = Array.from(line.matchAll(/[^?]+(?:\?|$)/g))
|
||||
.map((match) => normalizeQuestionChunk(match[0]))
|
||||
.filter((item) => item.length > 0);
|
||||
if (questionLike.length > 1) {
|
||||
for (const item of questionLike) {
|
||||
chunks.push(item.endsWith("?") ? item : `${item}?`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chunks.push(normalizeQuestionChunk(line));
|
||||
}
|
||||
return chunks.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function normalizeRuntimeQuestions(value: unknown): string[] {
|
||||
const raw = toArray(value)
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
if (raw.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const expanded = raw.flatMap((item) => splitQuestionCandidate(item));
|
||||
const deduped: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of expanded) {
|
||||
const normalized = normalizeQuestionChunk(item);
|
||||
if (!normalized) continue;
|
||||
if (seen.has(normalized)) continue;
|
||||
seen.add(normalized);
|
||||
deduped.push(normalized);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function normalizeCaseIds(value: unknown): string[] | undefined {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function buildEvalPayloadFromBody(body: Record<string, unknown>): {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
useMock: boolean;
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
rawQuestions?: string;
|
||||
evalTarget: EvalTarget;
|
||||
compareWithReportFile?: string;
|
||||
} {
|
||||
return {
|
||||
normalizeConfig: (body.normalizeConfig ?? {}) as Omit<NormalizeRequestPayload, "userQuestion" | "context">,
|
||||
caseIds: normalizeCaseIds(body.caseIds),
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: (body.mode as EvalRunMode | undefined) ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: (body.eval_target as EvalTarget | undefined) ?? "normalizer",
|
||||
compareWithReportFile:
|
||||
typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
|
||||
function resolveReadablePath(inputPath: string): string {
|
||||
if (path.isAbsolute(inputPath)) {
|
||||
return inputPath;
|
||||
}
|
||||
const candidates = [
|
||||
path.resolve(EVAL_CASES_DIR, inputPath),
|
||||
path.resolve(EVAL_DATASETS_DIR, inputPath),
|
||||
path.resolve(inputPath)
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
function readAssistantSuiteCaseSeeds(inputPath: string): Array<{ case_id: string; turns_total: number }> {
|
||||
const filePath = resolveReadablePath(inputPath);
|
||||
const raw = fs.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
const record = toRecord(parsed);
|
||||
const cases = toArray(record?.cases);
|
||||
return cases
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item): item is Record<string, unknown> => item !== null)
|
||||
.map((item) => {
|
||||
const caseId = toStringSafe(item.case_id);
|
||||
const turns = toArray(item.turns);
|
||||
if (!caseId || turns.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
case_id: caseId,
|
||||
turns_total: turns.length
|
||||
};
|
||||
})
|
||||
.filter((item): item is { case_id: string; turns_total: number } => item !== null);
|
||||
}
|
||||
|
||||
function writeRuntimeAssistantSuiteFromQuestions(jobId: string, questions: string[]): string {
|
||||
if (!fs.existsSync(EVAL_CASES_DIR)) {
|
||||
fs.mkdirSync(EVAL_CASES_DIR, { recursive: true });
|
||||
}
|
||||
const cases = questions.map((question, index) => {
|
||||
const caseId = `AUTO-${String(index + 1).padStart(3, "0")}`;
|
||||
return {
|
||||
case_id: caseId,
|
||||
scenario_tag: "autogen_runtime",
|
||||
question_type: "direct",
|
||||
broadness_level: "medium",
|
||||
turns: [{ user_message: question }]
|
||||
};
|
||||
});
|
||||
const payload = {
|
||||
suite_id: `assistant_autogen_runtime_${jobId}`,
|
||||
suite_version: "0.1.0",
|
||||
schema_version: "assistant_autogen_runtime_v0_1",
|
||||
scenario_count: cases.length,
|
||||
case_ids: cases.map((item) => item.case_id),
|
||||
cases
|
||||
};
|
||||
const fileName = `assistant_autogen_runtime_${jobId}.json`;
|
||||
fs.writeFileSync(path.resolve(EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
|
||||
return fileName;
|
||||
}
|
||||
|
||||
function readSessionConversation(runId: string, caseId: string): EvalAsyncCaseInfo["messages"] {
|
||||
const sessionId = `${runId}-${caseId}`;
|
||||
const filePath = path.resolve(ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
|
||||
const record = toRecord(parsed);
|
||||
const conversation = toArray(record?.conversation)
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item): item is Record<string, unknown> => item !== null);
|
||||
return conversation.map((item, index) => ({
|
||||
message_id: toStringSafe(item.message_id),
|
||||
role: toStringSafe(item.role) ?? "unknown",
|
||||
text: toStringSafe(item.text) ?? "",
|
||||
created_at: toStringSafe(item.created_at),
|
||||
trace_id: toStringSafe(item.trace_id),
|
||||
reply_type: toStringSafe(item.reply_type),
|
||||
message_index: index,
|
||||
case_id: caseId,
|
||||
case_message_index: index
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function syncJobWithSessions(job: EvalAsyncJob): void {
|
||||
if (!job.run_id || !job.eval_target.startsWith("assistant_")) {
|
||||
return;
|
||||
}
|
||||
let completed = 0;
|
||||
let hasRunning = false;
|
||||
for (const item of job.cases) {
|
||||
const messages = readSessionConversation(job.run_id, item.case_id);
|
||||
item.messages = messages;
|
||||
const assistantMessages = messages.filter((entry) => entry.role === "assistant").length;
|
||||
const userMessages = messages.filter((entry) => entry.role === "user").length;
|
||||
if (assistantMessages >= item.turns_total && item.turns_total > 0) {
|
||||
item.status = "completed";
|
||||
completed += 1;
|
||||
continue;
|
||||
}
|
||||
if (userMessages > 0 || messages.length > 0) {
|
||||
item.status = "running";
|
||||
hasRunning = true;
|
||||
continue;
|
||||
}
|
||||
item.status = "queued";
|
||||
}
|
||||
job.completed_cases = completed;
|
||||
if (job.status === "running" && !hasRunning && completed === job.total_cases && job.total_cases > 0) {
|
||||
job.status = "completed";
|
||||
}
|
||||
}
|
||||
|
||||
function trimAsyncJobsStore(): void {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS) return;
|
||||
const sorted = Array.from(ASYNC_JOBS.values()).sort((a, b) => Date.parse(a.updated_at) - Date.parse(b.updated_at));
|
||||
for (const item of sorted) {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS) break;
|
||||
ASYNC_JOBS.delete(item.job_id);
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotJob(job: EvalAsyncJob): Record<string, unknown> {
|
||||
return {
|
||||
job_id: job.job_id,
|
||||
status: job.status,
|
||||
created_at: job.created_at,
|
||||
updated_at: job.updated_at,
|
||||
eval_target: job.eval_target,
|
||||
run_id: job.run_id,
|
||||
case_set_file: job.case_set_file,
|
||||
total_cases: job.total_cases,
|
||||
completed_cases: job.completed_cases,
|
||||
error: job.error,
|
||||
cases: job.cases,
|
||||
report_summary: job.report
|
||||
? {
|
||||
run_id: toStringSafe(job.report.run_id),
|
||||
run_timestamp: toStringSafe(job.report.run_timestamp) ?? toStringSafe(job.report.timestamp),
|
||||
score_index:
|
||||
typeof job.report.score_index === "number"
|
||||
? Number(job.report.score_index)
|
||||
: toRecord(job.report.metrics) && typeof toRecord(job.report.metrics)?.score_index === "number"
|
||||
? Number(toRecord(job.report.metrics)?.score_index)
|
||||
: null,
|
||||
cases_total: typeof job.report.cases_total === "number" ? Number(job.report.cases_total) : null
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
|
||||
export function buildEvalRouter(services: AppServices): Router {
|
||||
const router = Router();
|
||||
|
||||
router.post("/api/eval/run", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {}) as Record<string, unknown>;
|
||||
const report = await services.evalService.run({
|
||||
normalizeConfig: (body.normalizeConfig ?? {}) as Omit<NormalizeRequestPayload, "userQuestion" | "context">,
|
||||
caseIds: Array.isArray(body.caseIds) ? (body.caseIds as string[]) : undefined,
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: (body.mode as EvalRunMode | undefined) ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: (body.eval_target as EvalTarget | undefined) ?? "normalizer",
|
||||
compareWithReportFile:
|
||||
typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
const report = await services.evalService.run(payload);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
report
|
||||
@@ -34,5 +337,115 @@ export function buildEvalRouter(services: AppServices): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/eval/run-async/start", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {}) as Record<string, unknown>;
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
if (payload.evalTarget !== "assistant_stage1") {
|
||||
throw new ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
|
||||
}
|
||||
const questions = normalizeRuntimeQuestions(body.questions);
|
||||
|
||||
const jobId = `job-${nanoid(10)}`;
|
||||
const runId = `assistant-stage1-${nanoid(10)}`;
|
||||
const runtimeCaseSetFile =
|
||||
questions.length > 0
|
||||
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
|
||||
: payload.caseSetFile
|
||||
? payload.caseSetFile
|
||||
: undefined;
|
||||
|
||||
if (!runtimeCaseSetFile) {
|
||||
throw new ApiError(
|
||||
"ASYNC_CASESET_REQUIRED",
|
||||
"Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.",
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
const caseSeeds = readAssistantSuiteCaseSeeds(runtimeCaseSetFile);
|
||||
if (caseSeeds.length === 0) {
|
||||
throw new ApiError("ASYNC_CASESET_EMPTY", "No runnable cases found in selected case-set.", 400);
|
||||
}
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const job: EvalAsyncJob = {
|
||||
job_id: jobId,
|
||||
status: "queued",
|
||||
created_at: nowIso,
|
||||
updated_at: nowIso,
|
||||
eval_target: payload.evalTarget,
|
||||
run_id: runId,
|
||||
case_set_file: runtimeCaseSetFile,
|
||||
total_cases: caseSeeds.length,
|
||||
completed_cases: 0,
|
||||
cases: caseSeeds.map((item) => ({
|
||||
case_id: item.case_id,
|
||||
turns_total: item.turns_total,
|
||||
status: "queued",
|
||||
messages: []
|
||||
})),
|
||||
error: null,
|
||||
report: null
|
||||
};
|
||||
ASYNC_JOBS.set(job.job_id, job);
|
||||
trimAsyncJobsStore();
|
||||
|
||||
setImmediate(() => {
|
||||
void (async () => {
|
||||
const target = ASYNC_JOBS.get(job.job_id);
|
||||
if (!target) return;
|
||||
target.status = "running";
|
||||
target.updated_at = new Date().toISOString();
|
||||
try {
|
||||
const report = await services.evalService.run({
|
||||
...payload,
|
||||
caseSetFile: runtimeCaseSetFile,
|
||||
runId
|
||||
});
|
||||
target.report = report;
|
||||
syncJobWithSessions(target);
|
||||
target.completed_cases = target.total_cases;
|
||||
target.status = "completed";
|
||||
target.updated_at = new Date().toISOString();
|
||||
} catch (error) {
|
||||
syncJobWithSessions(target);
|
||||
target.status = "failed";
|
||||
target.error = error instanceof Error ? error.message : String(error);
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
ok(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/api/eval/run-async/:job_id", (req, res, next) => {
|
||||
try {
|
||||
const jobId = String(req.params.job_id ?? "").trim();
|
||||
if (!jobId) {
|
||||
throw new ApiError("INVALID_ASYNC_JOB_ID", "job_id is required.", 400);
|
||||
}
|
||||
const job = ASYNC_JOBS.get(jobId);
|
||||
if (!job) {
|
||||
throw new ApiError("ASYNC_JOB_NOT_FOUND", `Async eval job not found: ${jobId}`, 404);
|
||||
}
|
||||
syncJobWithSessions(job);
|
||||
job.updated_at = new Date().toISOString();
|
||||
ok(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ export function createApp(): express.Express {
|
||||
app.use(buildNormalizeRouter(services));
|
||||
app.use(buildEvalRouter(services));
|
||||
app.use(buildAssistantRouter(services));
|
||||
app.use(buildAutoRunsRouter());
|
||||
app.use(buildAutoRunsRouter(openaiClient));
|
||||
app.use(buildHistoryRouter());
|
||||
app.use(buildPresetsRouter());
|
||||
app.use(buildAccountingAgentRouter(services));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// @ts-nocheck
|
||||
// @ts-nocheck
|
||||
import * as nanoid_1 from "nanoid";
|
||||
import * as stage1Contracts_1 from "../types/stage1Contracts";
|
||||
import * as config_1 from "../config";
|
||||
@@ -4789,6 +4789,10 @@ export class AssistantService {
|
||||
debug: null
|
||||
};
|
||||
this.sessions.appendItem(sessionId, userItem);
|
||||
const sessionAfterUserAppend = this.sessions.getSession(sessionId);
|
||||
if (sessionAfterUserAppend) {
|
||||
this.sessionLogger.persistSession(sessionAfterUserAppend);
|
||||
}
|
||||
const sessionOrganizationScope = resolveSessionOrganizationScopeContext(userMessage, session.items);
|
||||
const finalizeAddressLaneResponse = (addressLane, effectiveAddressUserMessage, carryoverMeta = null, llmPreDecomposeMeta = null) => {
|
||||
const safeAddressReply = sanitizeOutgoingAssistantText(addressLane.reply_text);
|
||||
|
||||
@@ -1876,6 +1876,7 @@ export class EvalService {
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
@@ -1887,7 +1888,7 @@ export class EvalService {
|
||||
|
||||
const suite = parseAssistantSuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage1-${nanoid(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage1-${nanoid(10)}`;
|
||||
const assistantService = new AssistantService(this.normalizerService, new AssistantSessionStore());
|
||||
const diagnostics: AssistantCaseDiagnostics[] = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1905,6 +1906,7 @@ export class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2223,6 +2225,7 @@ export class EvalService {
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
@@ -2234,7 +2237,7 @@ export class EvalService {
|
||||
|
||||
const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage2-${nanoid(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage2-${nanoid(10)}`;
|
||||
const assistantService = new AssistantService(this.normalizerService, new AssistantSessionStore());
|
||||
const diagnostics: AssistantStage2CaseDiagnostics[] = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -2255,6 +2258,7 @@ export class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2548,6 +2552,7 @@ export class EvalService {
|
||||
rawQuestions?: string;
|
||||
evalTarget?: EvalTarget;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const mode = payload.mode ?? "standard";
|
||||
const evalTarget = payload.evalTarget ?? "normalizer";
|
||||
@@ -2559,7 +2564,8 @@ export class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2570,7 +2576,8 @@ export class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user