АДРЕСНЫЙ РЕЖИМ - авторан история - юи + адресный рендер прогонов в реалтайме
This commit is contained in:
@@ -107,3 +107,60 @@ npm.cmd run dev:all
|
||||
cd X:\1C\NDC_1C\llm_normalizer\backend
|
||||
npm test
|
||||
```
|
||||
|
||||
## История автопрогонов и разметка
|
||||
|
||||
В интерфейсе есть отдельный режим `История автопрогонов` с операционным циклом:
|
||||
|
||||
1. Настроить генерацию вопросов (режим, количество, личность, prompt личности).
|
||||
2. Сгенерировать пачку вопросов.
|
||||
3. Отредактировать вопросы перед запуском.
|
||||
4. Запустить асинхронный прогон (`assistant_stage1`, `single-pass-strict`).
|
||||
5. Смотреть диалог прогона в live-режиме (polling статуса + сообщения по кейсам).
|
||||
6. Разметить ответы ассистента:
|
||||
- рейтинг `1..5`,
|
||||
- комментарий,
|
||||
- `manual_case_decision`,
|
||||
- автор.
|
||||
7. Отметить кейс выполненным (`resolved`) или вернуть в открытые.
|
||||
8. Смотреть пост-анализ и очереди фиксов по категориям.
|
||||
|
||||
### Важный момент по `qwen_seed`
|
||||
|
||||
`qwen_seed` использует тот же активный LLM-контур подключения, что и ответы ассистента
|
||||
(тот же provider/model/baseUrl), но в другой роли: генератор вопросов.
|
||||
|
||||
### Основные API для автопрогонов
|
||||
|
||||
- `GET /api/autoruns/history`
|
||||
- `GET /api/autoruns/history/:run_id`
|
||||
- `GET /api/autoruns/history/:run_id/case/:case_id/dialog`
|
||||
- `GET /api/autoruns/annotations`
|
||||
- `POST /api/autoruns/annotations`
|
||||
- `PATCH /api/autoruns/annotations/:annotation_id`
|
||||
- `GET /api/autoruns/manual-decision-schema`
|
||||
- `GET /api/autoruns/post-analysis`
|
||||
- `GET /api/autoruns/autogen/history`
|
||||
- `GET /api/autoruns/autogen/personality-catalog`
|
||||
- `POST /api/autoruns/autogen/generate`
|
||||
- `POST /api/eval/run-async/start`
|
||||
- `GET /api/eval/run-async/:job_id`
|
||||
|
||||
### Где лежат данные автопрогонов
|
||||
|
||||
- аннотации и ручная разметка:
|
||||
- `llm_normalizer/data/autorun_annotations/annotations.json`
|
||||
- история генераций:
|
||||
- `llm_normalizer/data/autorun_generators/history.json`
|
||||
- сгенерированные кейс-сеты (если включено сохранение):
|
||||
- `llm_normalizer/data/eval_cases/*.json`
|
||||
- сессии диалогов ассистента по кейсам:
|
||||
- `llm_normalizer/data/assistant_sessions/*.json`
|
||||
|
||||
### Канонические техдоки
|
||||
|
||||
- `docs/TECH/assistant_canon.md`
|
||||
- `docs/TECH/capabilities_registry.json`
|
||||
- `docs/TECH/manual_case_decision_schema.json`
|
||||
- `docs/TECH/ui_markup_system.md`
|
||||
- `docs/TECH/history_colibration.md`
|
||||
|
||||
+466
-15
@@ -7,9 +7,11 @@ exports.buildAutoRunsRouter = buildAutoRunsRouter;
|
||||
const fs_1 = __importDefault(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const express_1 = require("express");
|
||||
const iconv_lite_1 = __importDefault(require("iconv-lite"));
|
||||
const config_1 = require("../config");
|
||||
const http_1 = require("../utils/http");
|
||||
const capabilitiesRegistry_1 = require("../services/capabilitiesRegistry");
|
||||
const openaiResponsesClient_1 = require("../services/openaiResponsesClient");
|
||||
const MANUAL_CASE_DECISIONS = [
|
||||
"covered_ok",
|
||||
"covered_but_bad_answer",
|
||||
@@ -102,6 +104,10 @@ function parseAnnotationAuthor(value) {
|
||||
return null;
|
||||
return author.slice(0, 80);
|
||||
}
|
||||
function parseAnnotationResolved(value, fallback = false) {
|
||||
const parsed = toBooleanSafe(value);
|
||||
return parsed === null ? fallback : parsed;
|
||||
}
|
||||
function readManualDecisionSchema() {
|
||||
const fallback = {
|
||||
schema_version: "manual_case_decision_schema_v1_fallback",
|
||||
@@ -150,6 +156,8 @@ function readAutoGenHistory() {
|
||||
questions: toArray(item.questions)
|
||||
.map((q) => toStringSafe(q))
|
||||
.filter((q) => q !== null)
|
||||
.map((q) => sanitizeGeneratedQuestion(q))
|
||||
.filter((q) => q.length > 0)
|
||||
.slice(0, 500),
|
||||
generated_by: toStringSafe(item.generated_by),
|
||||
saved_case_set_file: toStringSafe(item.saved_case_set_file),
|
||||
@@ -160,6 +168,12 @@ function readAutoGenHistory() {
|
||||
assistant_prompt_version: toStringSafe(toRecord(item.context)?.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(toRecord(item.context)?.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(toRecord(item.context)?.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
}))
|
||||
@@ -207,11 +221,11 @@ function collectCanonicalQuestions(limit = 300) {
|
||||
for (const testCase of cases) {
|
||||
const rawQuestion = toStringSafe(testCase.raw_question) ?? toStringSafe(testCase.user_message) ?? toStringSafe(testCase.query);
|
||||
if (rawQuestion) {
|
||||
questions.push(rawQuestion);
|
||||
questions.push(sanitizeGeneratedQuestion(rawQuestion));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(new Set(questions)).slice(0, limit);
|
||||
return Array.from(new Set(questions.filter((item) => item.length > 0))).slice(0, limit);
|
||||
}
|
||||
function normalizeDomainHint(value) {
|
||||
const domain = toStringSafe(value);
|
||||
@@ -219,6 +233,49 @@ function normalizeDomainHint(value) {
|
||||
return null;
|
||||
return domain.toLowerCase();
|
||||
}
|
||||
function buildAutogenPromptFromCapabilityGroup(group) {
|
||||
const supported = group.supported_operations.slice(0, 3).join(", ");
|
||||
const examples = group.typical_queries.slice(0, 2).join(" | ");
|
||||
const hints = group.one_c_hints.slice(0, 2).join(", ");
|
||||
const operationsPart = supported ? ` Опирайся на операции: ${supported}.` : "";
|
||||
const examplesPart = examples ? ` Ближайшие формулировки: ${examples}.` : "";
|
||||
const hintsPart = hints ? ` Можно мягко упоминать контекст 1С: ${hints}.` : "";
|
||||
return (`Генерируй реалистичные вопросы бухгалтера по группе "${group.group_title}".` +
|
||||
` Добавляй живую разговорную форму и опечатки, но сохраняй бизнес-смысл.${operationsPart}${examplesPart}${hintsPart}` +
|
||||
" Не выдумывай операции вне read-only режима.");
|
||||
}
|
||||
function buildAutogenPersonalityCatalog() {
|
||||
const builtIn = [
|
||||
{
|
||||
id: "general",
|
||||
label: "Общий контур",
|
||||
domain: null,
|
||||
default_prompt: "Генерируй реалистичные живые вопросы бухгалтера по 1С. Добавляй разговорные формулировки и опечатки, но сохраняй бизнес-смысл.",
|
||||
source: "built_in"
|
||||
}
|
||||
];
|
||||
const registry = (0, capabilitiesRegistry_1.loadCapabilitiesRegistry)();
|
||||
const registryBased = registry.groups.map((group) => ({
|
||||
id: `registry_${group.group_code}`,
|
||||
label: `${group.group_title} (реестр)`,
|
||||
domain: group.group_code,
|
||||
default_prompt: buildAutogenPromptFromCapabilityGroup(group),
|
||||
source: "capabilities_registry"
|
||||
}));
|
||||
const dedup = new Map();
|
||||
for (const item of [...builtIn, ...registryBased]) {
|
||||
if (!item.id.trim())
|
||||
continue;
|
||||
if (!dedup.has(item.id)) {
|
||||
dedup.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return [...dedup.values()].map((item) => ({
|
||||
...item,
|
||||
label: repairAutogenMojibake(item.label),
|
||||
default_prompt: repairAutogenMojibake(item.default_prompt)
|
||||
}));
|
||||
}
|
||||
function fallbackDomainTemplates(domain) {
|
||||
if (domain?.includes("vat") || domain?.includes("ндс")) {
|
||||
return [
|
||||
@@ -276,9 +333,9 @@ function generateQwenSeedQuestions(count, domain) {
|
||||
const out = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = bag[index % bag.length];
|
||||
out.push(mutateIntoQwenStyle(base, index));
|
||||
out.push(sanitizeGeneratedQuestion(mutateIntoQwenStyle(base, index)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
function generateCodexCreativeQuestions(count, domain) {
|
||||
const domainTemplates = fallbackDomainTemplates(domain);
|
||||
@@ -293,9 +350,9 @@ function generateCodexCreativeQuestions(count, domain) {
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = domainTemplates[index % domainTemplates.length];
|
||||
const pattern = patterns[index % patterns.length];
|
||||
out.push(pattern.replace("{q}", base));
|
||||
out.push(sanitizeGeneratedQuestion(pattern.replace("{q}", base)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
function generateAutogenId() {
|
||||
return `gen-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
@@ -325,6 +382,9 @@ function readAnnotations() {
|
||||
comment: toStringSafe(item.comment) ?? "",
|
||||
manual_case_decision: parseManualCaseDecision(item.manual_case_decision),
|
||||
annotation_author: parseAnnotationAuthor(item.annotation_author),
|
||||
resolved: parseAnnotationResolved(item.resolved),
|
||||
resolved_at: toStringSafe(item.resolved_at),
|
||||
resolved_by: parseAnnotationAuthor(item.resolved_by),
|
||||
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
|
||||
updated_at: toStringSafe(item.updated_at) ?? new Date().toISOString(),
|
||||
context: {
|
||||
@@ -334,7 +394,9 @@ function readAnnotations() {
|
||||
eval_target: toStringSafe(context?.eval_target) ?? "unknown",
|
||||
prompt_version: toStringSafe(context?.prompt_version),
|
||||
domain: toStringSafe(context?.domain),
|
||||
query_class: toStringSafe(context?.query_class)
|
||||
query_class: toStringSafe(context?.query_class),
|
||||
question_text: toStringSafe(context?.question_text),
|
||||
answer_text: toStringSafe(context?.answer_text)
|
||||
}
|
||||
};
|
||||
})
|
||||
@@ -946,6 +1008,37 @@ function withMessageAnnotations(runId, caseId, messages, annotations) {
|
||||
};
|
||||
});
|
||||
}
|
||||
function buildRunAggregateDialog(run, annotations) {
|
||||
const cases = buildCaseSummaries(run.report, run.run_id, false);
|
||||
const messages = [];
|
||||
const decomposition = [];
|
||||
let globalMessageIndex = 0;
|
||||
for (const item of cases) {
|
||||
const caseId = item.case_id;
|
||||
const caseDialog = loadSessionDialog(run.run_id, caseId) ?? buildFallbackDialog(run, caseId);
|
||||
const annotatedCaseMessages = withMessageAnnotations(run.run_id, caseId, caseDialog.messages, annotations);
|
||||
for (const caseMessage of annotatedCaseMessages) {
|
||||
const localMessageIndex = toNumberSafe(caseMessage.message_index) ?? 0;
|
||||
messages.push({
|
||||
...caseMessage,
|
||||
case_id: caseId,
|
||||
case_message_index: localMessageIndex,
|
||||
message_index: globalMessageIndex
|
||||
});
|
||||
globalMessageIndex += 1;
|
||||
}
|
||||
if (caseDialog.decomposition.length > 0) {
|
||||
decomposition.push(...caseDialog.decomposition.map((step) => `[${caseId}] ${step}`));
|
||||
}
|
||||
}
|
||||
return {
|
||||
source: "run_aggregate",
|
||||
session_id: `${run.run_id}::__all__`,
|
||||
messages,
|
||||
decomposition,
|
||||
assistant_mode: null
|
||||
};
|
||||
}
|
||||
function generateAnnotationId() {
|
||||
return `ann-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
}
|
||||
@@ -975,6 +1068,265 @@ function parseAutogenDomain(value) {
|
||||
return null;
|
||||
return domain.slice(0, 80);
|
||||
}
|
||||
function parseAutogenLlmRuntimeConfig(body, context) {
|
||||
const llm = toRecord(body.llm);
|
||||
const providerRaw = toStringSafe(llm?.llm_provider ?? context?.llm_provider)?.toLowerCase() ?? "";
|
||||
const model = toStringSafe(llm?.model ?? context?.model);
|
||||
if (!model || (providerRaw !== "openai" && providerRaw !== "local")) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
llm_provider: providerRaw === "local" ? "local" : "openai",
|
||||
api_key: toStringSafe(llm?.api_key) ?? "",
|
||||
model,
|
||||
base_url: toStringSafe(llm?.base_url),
|
||||
temperature: toNumberSafe(llm?.temperature),
|
||||
max_output_tokens: toNumberSafe(llm?.max_output_tokens)
|
||||
};
|
||||
}
|
||||
function textMojibakeScore(value) {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
|
||||
}
|
||||
function looksLikeMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
|
||||
}
|
||||
function repairAutogenMojibake(value) {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibake(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
let improved = false;
|
||||
try {
|
||||
const fromWin1251 = iconv_lite_1.default.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScore(fromLatin1) > textMojibakeScore(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
improved = true;
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore
|
||||
}
|
||||
if (!improved) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
function sanitizeGeneratedQuestion(value) {
|
||||
return repairAutogenMojibake(String(value ?? ""))
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
function splitQuestionCandidates(rawText) {
|
||||
const normalized = repairAutogenMojibake(rawText).replace(/\r/g, "\n").trim();
|
||||
if (!normalized)
|
||||
return [];
|
||||
const unescaped = normalized.replace(/\\"/g, '"').replace(/\\n/g, "\n");
|
||||
const byLines = unescaped
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, ""))
|
||||
.map((line) => sanitizeGeneratedQuestion(line))
|
||||
.filter((line) => line.length > 0);
|
||||
if (byLines.length > 1) {
|
||||
return byLines;
|
||||
}
|
||||
const questionMarkCount = (unescaped.match(/\?/g) ?? []).length;
|
||||
if (questionMarkCount > 1) {
|
||||
const byQuestion = unescaped
|
||||
.split("?")
|
||||
.map((chunk) => sanitizeGeneratedQuestion(chunk))
|
||||
.filter((chunk) => chunk.length > 0)
|
||||
.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`));
|
||||
if (byQuestion.length > 1) {
|
||||
return byQuestion;
|
||||
}
|
||||
}
|
||||
const quoted = Array.from(unescaped.matchAll(/"([^"\n]{6,}?)"/g))
|
||||
.map((match) => sanitizeGeneratedQuestion(match[1]))
|
||||
.filter((line) => line.length > 0);
|
||||
if (quoted.length > 1) {
|
||||
return quoted;
|
||||
}
|
||||
const cleaned = sanitizeGeneratedQuestion(unescaped);
|
||||
return cleaned ? [cleaned] : [];
|
||||
}
|
||||
function parseAutogenOutputJson(rawText) {
|
||||
const cleaned = repairAutogenMojibake(rawText)
|
||||
.trim()
|
||||
.replace(/^```json\s*/i, "")
|
||||
.replace(/^```\s*/i, "")
|
||||
.replace(/```$/i, "")
|
||||
.trim();
|
||||
if (!cleaned)
|
||||
return null;
|
||||
try {
|
||||
return JSON.parse(cleaned);
|
||||
}
|
||||
catch {
|
||||
// continue
|
||||
}
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const arrayEnd = cleaned.lastIndexOf("]");
|
||||
if (arrayStart >= 0 && arrayEnd > arrayStart) {
|
||||
const fragment = cleaned.slice(arrayStart, arrayEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment);
|
||||
}
|
||||
catch {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
const objStart = cleaned.indexOf("{");
|
||||
const objEnd = cleaned.lastIndexOf("}");
|
||||
if (objStart >= 0 && objEnd > objStart) {
|
||||
const fragment = cleaned.slice(objStart, objEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment);
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function collectQuestionsFromCandidate(value, depth = 0) {
|
||||
if (depth > 5 || value === null || value === undefined) {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
const text = value.trim();
|
||||
if (!text)
|
||||
return [];
|
||||
const nestedParsed = parseAutogenOutputJson(text);
|
||||
if (nestedParsed !== null) {
|
||||
const nestedQuestions = collectQuestionsFromCandidate(nestedParsed, depth + 1);
|
||||
if (nestedQuestions.length > 0) {
|
||||
return nestedQuestions;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const decoded = JSON.parse(text);
|
||||
if (decoded !== text) {
|
||||
const decodedQuestions = collectQuestionsFromCandidate(decoded, depth + 1);
|
||||
if (decodedQuestions.length > 0) {
|
||||
return decodedQuestions;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// ignore non-JSON strings
|
||||
}
|
||||
return splitQuestionCandidates(text);
|
||||
}
|
||||
const record = toRecord(value);
|
||||
if (!record) {
|
||||
return [];
|
||||
}
|
||||
const fromQuestions = collectQuestionsFromCandidate(record.questions, depth + 1);
|
||||
if (fromQuestions.length > 0) {
|
||||
return fromQuestions;
|
||||
}
|
||||
const fallbackText = toStringSafe(record.question ?? record.user_message ?? record.text);
|
||||
return fallbackText ? splitQuestionCandidates(fallbackText) : [];
|
||||
}
|
||||
function extractQuestionsFromAutogenOutput(rawText) {
|
||||
const parsed = parseAutogenOutputJson(rawText);
|
||||
const fromParsed = collectQuestionsFromCandidate(parsed);
|
||||
if (fromParsed.length > 0) {
|
||||
return fromParsed;
|
||||
}
|
||||
return collectQuestionsFromCandidate(rawText);
|
||||
}
|
||||
async function generateQwenSeedQuestionsLive(input) {
|
||||
const seedExamples = collectCanonicalQuestions(40);
|
||||
const fallbackExamples = fallbackDomainTemplates(input.domain);
|
||||
const examples = (seedExamples.length > 0 ? seedExamples : fallbackExamples).slice(0, 8);
|
||||
const personalityPrompt = input.personalityPrompt ??
|
||||
"Генерируй реалистичные вопросы бухгалтера по 1С. Разговорный стиль допустим, но смысл должен быть четким.";
|
||||
const repairedPersonalityPrompt = repairAutogenMojibake(personalityPrompt);
|
||||
const maxOutputTokens = clampInt(input.llmConfig.max_output_tokens, 300, 3000, 1200);
|
||||
const temperature = input.llmConfig.temperature === null ? 0.5 : Math.max(0, Math.min(1.5, input.llmConfig.temperature));
|
||||
const systemPrompt = [
|
||||
"Ты генератор вопросов для автопрогонов бухгалтерского ассистента по 1С.",
|
||||
"Возвращай только JSON и никаких пояснений.",
|
||||
"Ассистент работает в read-only режиме: не проси действий изменения базы."
|
||||
].join(" ");
|
||||
const repairedSystemPrompt = repairAutogenMojibake(systemPrompt);
|
||||
const developerPrompt = [
|
||||
`Нужно сгенерировать ровно ${input.count} вопросов.`,
|
||||
"Формат ответа строго:",
|
||||
'{"questions":["вопрос 1","вопрос 2"]}',
|
||||
"Требования:",
|
||||
"1) каждый вопрос отдельный, без дубликатов;",
|
||||
"2) живой пользовательский язык;",
|
||||
"3) допустимы легкие разговорные сокращения;",
|
||||
"4) не выдавай мета-комментарии и не описывай правила."
|
||||
].join("\n");
|
||||
const repairedDeveloperPrompt = repairAutogenMojibake(developerPrompt);
|
||||
const userMessage = [
|
||||
`Домен: ${input.domain ?? "general"}.`,
|
||||
`Промпт личности: ${repairedPersonalityPrompt}`,
|
||||
"Примеры ориентиров по стилю и тематике:",
|
||||
...examples.map((item, index) => `${index + 1}. ${item}`)
|
||||
].join("\n");
|
||||
const repairedUserMessage = repairAutogenMojibake(userMessage);
|
||||
const response = await input.client.chat({
|
||||
llmProvider: input.llmConfig.llm_provider,
|
||||
apiKey: input.llmConfig.api_key,
|
||||
model: input.llmConfig.model,
|
||||
baseUrl: input.llmConfig.base_url ?? undefined,
|
||||
temperature,
|
||||
maxOutputTokens: maxOutputTokens
|
||||
}, {
|
||||
systemPrompt: repairedSystemPrompt,
|
||||
developerPrompt: repairedDeveloperPrompt,
|
||||
userMessage: repairedUserMessage,
|
||||
temperature,
|
||||
maxOutputTokens
|
||||
});
|
||||
const extracted = extractQuestionsFromAutogenOutput(response.outputText);
|
||||
const normalized = Array.from(new Set(extracted.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
if (normalized.length === 0) {
|
||||
throw new http_1.ApiError("AUTOGEN_LLM_EMPTY_OUTPUT", "Qwen не вернул пригодные вопросы для автогенерации.", 502, {
|
||||
model: input.llmConfig.model
|
||||
});
|
||||
}
|
||||
const fallback = generateQwenSeedQuestions(input.count, input.domain);
|
||||
return Array.from(new Set([...normalized, ...fallback])).slice(0, input.count);
|
||||
}
|
||||
function hasAnyRunFilterQuery(query) {
|
||||
return Boolean(toStringSafe(query.from) ??
|
||||
toStringSafe(query.to) ??
|
||||
@@ -996,7 +1348,8 @@ function buildAutogenCaseSetFileName(mode, generationId) {
|
||||
return `assistant_autogen_${mode}_${stamp}_${generationId}.json`;
|
||||
}
|
||||
function buildAutogenCaseSetPayload(input) {
|
||||
const cases = input.questions.map((question, index) => ({
|
||||
const normalizedQuestions = Array.from(new Set(input.questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
const cases = normalizedQuestions.map((question, index) => ({
|
||||
case_id: `AUTO-${String(index + 1).padStart(3, "0")}`,
|
||||
scenario_tag: `${input.mode}_${input.domain ?? "general"}`,
|
||||
question_type: "direct",
|
||||
@@ -1103,7 +1456,7 @@ function collectPostAnalysis(annotations, runMap, limitPerQueue) {
|
||||
].slice(0, 60)
|
||||
};
|
||||
}
|
||||
function buildAutoRunsRouter() {
|
||||
function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIResponsesClient()) {
|
||||
const router = (0, express_1.Router)();
|
||||
router.get("/api/autoruns/history", (req, res) => {
|
||||
const filters = parseFilters(req.query);
|
||||
@@ -1175,9 +1528,22 @@ function buildAutoRunsRouter() {
|
||||
if (!run) {
|
||||
throw new http_1.ApiError("AUTORUN_NOT_FOUND", `Run not found: ${runId}`, 404);
|
||||
}
|
||||
const annotations = readAnnotations();
|
||||
if (caseId === "__all__") {
|
||||
const dialog = buildRunAggregateDialog(run, annotations);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
run_id: runId,
|
||||
case_id: "__all__",
|
||||
...dialog,
|
||||
annotations: annotations
|
||||
.filter((item) => item.run_id === runId)
|
||||
.sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))
|
||||
});
|
||||
return;
|
||||
}
|
||||
const sessionDialog = loadSessionDialog(runId, caseId);
|
||||
const dialog = sessionDialog ?? buildFallbackDialog(run, caseId);
|
||||
const annotations = readAnnotations();
|
||||
const messages = withMessageAnnotations(runId, caseId, dialog.messages, annotations);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
@@ -1307,6 +1673,9 @@ function buildAutoRunsRouter() {
|
||||
if (targetRole !== "assistant") {
|
||||
throw new http_1.ApiError("AUTORUN_MESSAGE_NOT_ASSISTANT", "Only assistant answers can be annotated", 400);
|
||||
}
|
||||
const pairedUserQuestion = [...dialog.messages.slice(0, messageIndex)]
|
||||
.reverse()
|
||||
.find((item) => (toStringSafe(item.role) ?? "") === "user");
|
||||
const nowIso = new Date().toISOString();
|
||||
const annotations = readAnnotations();
|
||||
const key = annotationKey(runId, caseId, messageIndex);
|
||||
@@ -1322,6 +1691,9 @@ function buildAutoRunsRouter() {
|
||||
comment,
|
||||
manual_case_decision: manualCaseDecision,
|
||||
annotation_author: annotationAuthor,
|
||||
resolved: existing?.resolved ?? false,
|
||||
resolved_at: existing?.resolved_at ?? null,
|
||||
resolved_by: existing?.resolved_by ?? null,
|
||||
created_at: existing?.created_at ?? nowIso,
|
||||
updated_at: nowIso,
|
||||
context: {
|
||||
@@ -1331,7 +1703,9 @@ function buildAutoRunsRouter() {
|
||||
eval_target: run.eval_target,
|
||||
prompt_version: toStringSafe(run.report.prompt_version),
|
||||
domain: caseSummary.domain,
|
||||
query_class: caseSummary.query_class
|
||||
query_class: caseSummary.query_class,
|
||||
question_text: toStringSafe(pairedUserQuestion?.text),
|
||||
answer_text: toStringSafe(targetMessage.text)
|
||||
}
|
||||
};
|
||||
if (existingIndex >= 0) {
|
||||
@@ -1353,6 +1727,49 @@ function buildAutoRunsRouter() {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.patch("/api/autoruns/annotations/:annotation_id", (req, res, next) => {
|
||||
try {
|
||||
const annotationId = toStringSafe(req.params.annotation_id);
|
||||
if (!annotationId) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_ID", "annotation_id is required", 400);
|
||||
}
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_PATCH", "JSON body is required", 400);
|
||||
}
|
||||
const resolved = toBooleanSafe(body.resolved);
|
||||
if (resolved === null) {
|
||||
throw new http_1.ApiError("INVALID_ANNOTATION_PATCH", "resolved flag is required", 400);
|
||||
}
|
||||
const resolvedBy = parseAnnotationAuthor(body.resolved_by);
|
||||
const annotations = readAnnotations();
|
||||
const index = annotations.findIndex((item) => item.annotation_id === annotationId);
|
||||
if (index < 0) {
|
||||
throw new http_1.ApiError("ANNOTATION_NOT_FOUND", `Annotation not found: ${annotationId}`, 404);
|
||||
}
|
||||
const nowIso = new Date().toISOString();
|
||||
const current = annotations[index];
|
||||
const updated = {
|
||||
...current,
|
||||
resolved,
|
||||
resolved_at: resolved ? nowIso : null,
|
||||
resolved_by: resolved ? resolvedBy ?? current.resolved_by ?? null : null,
|
||||
updated_at: nowIso
|
||||
};
|
||||
annotations[index] = updated;
|
||||
writeAnnotations(annotations);
|
||||
const statsByCase = buildAnnotationStatsMap(updated.run_id, annotations);
|
||||
const caseStats = statsByCase.get(updated.case_id) ?? null;
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
annotation: updated,
|
||||
case_annotation_stats: caseStats
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.get("/api/autoruns/manual-decision-schema", (_req, res) => {
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
@@ -1416,7 +1833,19 @@ function buildAutoRunsRouter() {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/autoruns/autogen/generate", (req, res, next) => {
|
||||
router.get("/api/autoruns/autogen/personality-catalog", (_req, res, next) => {
|
||||
try {
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
generated_at: new Date().toISOString(),
|
||||
items: buildAutogenPersonalityCatalog()
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
|
||||
try {
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
@@ -1428,9 +1857,25 @@ function buildAutoRunsRouter() {
|
||||
const persistCaseSet = toBooleanSafe(body.persist_to_eval_cases) ?? true;
|
||||
const generatedBy = parseAnnotationAuthor(body.generated_by);
|
||||
const context = toRecord(body.context);
|
||||
const questions = mode === "qwen_seed"
|
||||
? generateQwenSeedQuestions(count, domain)
|
||||
: generateCodexCreativeQuestions(count, domain);
|
||||
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
|
||||
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
|
||||
let questions = [];
|
||||
if (mode === "qwen_seed") {
|
||||
if (!llmConfig) {
|
||||
throw new http_1.ApiError("AUTOGEN_LLM_CONFIG_REQUIRED", "Для режима qwen_seed нужен активный LLM-контур (provider/model/baseUrl) из настроек подключения.", 400);
|
||||
}
|
||||
questions = await generateQwenSeedQuestionsLive({
|
||||
count,
|
||||
domain,
|
||||
personalityPrompt,
|
||||
llmConfig,
|
||||
client: openaiClient
|
||||
});
|
||||
}
|
||||
else {
|
||||
questions = generateCodexCreativeQuestions(count, domain);
|
||||
}
|
||||
questions = Array.from(new Set(questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))).slice(0, count);
|
||||
const generationId = generateAutogenId();
|
||||
let savedCaseSetFile = null;
|
||||
if (persistCaseSet) {
|
||||
@@ -1464,6 +1909,12 @@ function buildAutoRunsRouter() {
|
||||
assistant_prompt_version: toStringSafe(context.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(context.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(context.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(context.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(context.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(context.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
};
|
||||
|
||||
+363
-14
@@ -1,27 +1,275 @@
|
||||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.buildEvalRouter = buildEvalRouter;
|
||||
const fs_1 = __importDefault(require("fs"));
|
||||
const path_1 = __importDefault(require("path"));
|
||||
const nanoid_1 = require("nanoid");
|
||||
const express_1 = require("express");
|
||||
const config_1 = require("../config");
|
||||
const http_1 = require("../utils/http");
|
||||
const ASYNC_JOBS = new Map();
|
||||
const MAX_ASYNC_JOBS = 80;
|
||||
function toRecord(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
function toStringSafe(value) {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
function toArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
function normalizeQuestionChunk(value) {
|
||||
return String(value ?? "")
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
function splitQuestionCandidate(raw) {
|
||||
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
const byLines = normalized
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, "").trim())
|
||||
.filter((line) => line.length > 0);
|
||||
const source = byLines.length > 1 ? byLines : [normalized];
|
||||
const chunks = [];
|
||||
for (const line of source) {
|
||||
const questionLike = Array.from(line.matchAll(/[^?]+(?:\?|$)/g))
|
||||
.map((match) => normalizeQuestionChunk(match[0]))
|
||||
.filter((item) => item.length > 0);
|
||||
if (questionLike.length > 1) {
|
||||
for (const item of questionLike) {
|
||||
chunks.push(item.endsWith("?") ? item : `${item}?`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chunks.push(normalizeQuestionChunk(line));
|
||||
}
|
||||
return chunks.filter((item) => item.length > 0);
|
||||
}
|
||||
function normalizeRuntimeQuestions(value) {
|
||||
const raw = toArray(value)
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
if (raw.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const expanded = raw.flatMap((item) => splitQuestionCandidate(item));
|
||||
const deduped = [];
|
||||
const seen = new Set();
|
||||
for (const item of expanded) {
|
||||
const normalized = normalizeQuestionChunk(item);
|
||||
if (!normalized)
|
||||
continue;
|
||||
if (seen.has(normalized))
|
||||
continue;
|
||||
seen.add(normalized);
|
||||
deduped.push(normalized);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
function normalizeCaseIds(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
function buildEvalPayloadFromBody(body) {
|
||||
return {
|
||||
normalizeConfig: (body.normalizeConfig ?? {}),
|
||||
caseIds: normalizeCaseIds(body.caseIds),
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: body.mode ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: body.eval_target ?? "normalizer",
|
||||
compareWithReportFile: typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
function resolveReadablePath(inputPath) {
|
||||
if (path_1.default.isAbsolute(inputPath)) {
|
||||
return inputPath;
|
||||
}
|
||||
const candidates = [
|
||||
path_1.default.resolve(config_1.EVAL_CASES_DIR, inputPath),
|
||||
path_1.default.resolve(config_1.EVAL_DATASETS_DIR, inputPath),
|
||||
path_1.default.resolve(inputPath)
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs_1.default.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
function readAssistantSuiteCaseSeeds(inputPath) {
|
||||
const filePath = resolveReadablePath(inputPath);
|
||||
const raw = fs_1.default.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
|
||||
const parsed = JSON.parse(raw);
|
||||
const record = toRecord(parsed);
|
||||
const cases = toArray(record?.cases);
|
||||
return cases
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item) => item !== null)
|
||||
.map((item) => {
|
||||
const caseId = toStringSafe(item.case_id);
|
||||
const turns = toArray(item.turns);
|
||||
if (!caseId || turns.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
case_id: caseId,
|
||||
turns_total: turns.length
|
||||
};
|
||||
})
|
||||
.filter((item) => item !== null);
|
||||
}
|
||||
function writeRuntimeAssistantSuiteFromQuestions(jobId, questions) {
|
||||
if (!fs_1.default.existsSync(config_1.EVAL_CASES_DIR)) {
|
||||
fs_1.default.mkdirSync(config_1.EVAL_CASES_DIR, { recursive: true });
|
||||
}
|
||||
const cases = questions.map((question, index) => {
|
||||
const caseId = `AUTO-${String(index + 1).padStart(3, "0")}`;
|
||||
return {
|
||||
case_id: caseId,
|
||||
scenario_tag: "autogen_runtime",
|
||||
question_type: "direct",
|
||||
broadness_level: "medium",
|
||||
turns: [{ user_message: question }]
|
||||
};
|
||||
});
|
||||
const payload = {
|
||||
suite_id: `assistant_autogen_runtime_${jobId}`,
|
||||
suite_version: "0.1.0",
|
||||
schema_version: "assistant_autogen_runtime_v0_1",
|
||||
scenario_count: cases.length,
|
||||
case_ids: cases.map((item) => item.case_id),
|
||||
cases
|
||||
};
|
||||
const fileName = `assistant_autogen_runtime_${jobId}.json`;
|
||||
fs_1.default.writeFileSync(path_1.default.resolve(config_1.EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
|
||||
return fileName;
|
||||
}
|
||||
function readSessionConversation(runId, caseId) {
|
||||
const sessionId = `${runId}-${caseId}`;
|
||||
const filePath = path_1.default.resolve(config_1.ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
|
||||
if (!fs_1.default.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(fs_1.default.readFileSync(filePath, "utf-8"));
|
||||
const record = toRecord(parsed);
|
||||
const conversation = toArray(record?.conversation)
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item) => item !== null);
|
||||
return conversation.map((item, index) => ({
|
||||
message_id: toStringSafe(item.message_id),
|
||||
role: toStringSafe(item.role) ?? "unknown",
|
||||
text: toStringSafe(item.text) ?? "",
|
||||
created_at: toStringSafe(item.created_at),
|
||||
trace_id: toStringSafe(item.trace_id),
|
||||
reply_type: toStringSafe(item.reply_type),
|
||||
message_index: index,
|
||||
case_id: caseId,
|
||||
case_message_index: index
|
||||
}));
|
||||
}
|
||||
catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
function syncJobWithSessions(job) {
|
||||
if (!job.run_id || !job.eval_target.startsWith("assistant_")) {
|
||||
return;
|
||||
}
|
||||
let completed = 0;
|
||||
let hasRunning = false;
|
||||
for (const item of job.cases) {
|
||||
const messages = readSessionConversation(job.run_id, item.case_id);
|
||||
item.messages = messages;
|
||||
const assistantMessages = messages.filter((entry) => entry.role === "assistant").length;
|
||||
const userMessages = messages.filter((entry) => entry.role === "user").length;
|
||||
if (assistantMessages >= item.turns_total && item.turns_total > 0) {
|
||||
item.status = "completed";
|
||||
completed += 1;
|
||||
continue;
|
||||
}
|
||||
if (userMessages > 0 || messages.length > 0) {
|
||||
item.status = "running";
|
||||
hasRunning = true;
|
||||
continue;
|
||||
}
|
||||
item.status = "queued";
|
||||
}
|
||||
job.completed_cases = completed;
|
||||
if (job.status === "running" && !hasRunning && completed === job.total_cases && job.total_cases > 0) {
|
||||
job.status = "completed";
|
||||
}
|
||||
}
|
||||
function trimAsyncJobsStore() {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS)
|
||||
return;
|
||||
const sorted = Array.from(ASYNC_JOBS.values()).sort((a, b) => Date.parse(a.updated_at) - Date.parse(b.updated_at));
|
||||
for (const item of sorted) {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS)
|
||||
break;
|
||||
ASYNC_JOBS.delete(item.job_id);
|
||||
}
|
||||
}
|
||||
function snapshotJob(job) {
|
||||
return {
|
||||
job_id: job.job_id,
|
||||
status: job.status,
|
||||
created_at: job.created_at,
|
||||
updated_at: job.updated_at,
|
||||
eval_target: job.eval_target,
|
||||
run_id: job.run_id,
|
||||
case_set_file: job.case_set_file,
|
||||
total_cases: job.total_cases,
|
||||
completed_cases: job.completed_cases,
|
||||
error: job.error,
|
||||
cases: job.cases,
|
||||
report_summary: job.report
|
||||
? {
|
||||
run_id: toStringSafe(job.report.run_id),
|
||||
run_timestamp: toStringSafe(job.report.run_timestamp) ?? toStringSafe(job.report.timestamp),
|
||||
score_index: typeof job.report.score_index === "number"
|
||||
? Number(job.report.score_index)
|
||||
: toRecord(job.report.metrics) && typeof toRecord(job.report.metrics)?.score_index === "number"
|
||||
? Number(toRecord(job.report.metrics)?.score_index)
|
||||
: null,
|
||||
cases_total: typeof job.report.cases_total === "number" ? Number(job.report.cases_total) : null
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
function buildEvalRouter(services) {
|
||||
const router = (0, express_1.Router)();
|
||||
router.post("/api/eval/run", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {});
|
||||
const report = await services.evalService.run({
|
||||
normalizeConfig: (body.normalizeConfig ?? {}),
|
||||
caseIds: Array.isArray(body.caseIds) ? body.caseIds : undefined,
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: body.mode ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: body.eval_target ?? "normalizer",
|
||||
compareWithReportFile: typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
const report = await services.evalService.run(payload);
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
report
|
||||
@@ -31,5 +279,106 @@ function buildEvalRouter(services) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.post("/api/eval/run-async/start", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
if (payload.evalTarget !== "assistant_stage1") {
|
||||
throw new http_1.ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
|
||||
}
|
||||
const questions = normalizeRuntimeQuestions(body.questions);
|
||||
const jobId = `job-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runtimeCaseSetFile = questions.length > 0
|
||||
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
|
||||
: payload.caseSetFile
|
||||
? payload.caseSetFile
|
||||
: undefined;
|
||||
if (!runtimeCaseSetFile) {
|
||||
throw new http_1.ApiError("ASYNC_CASESET_REQUIRED", "Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.", 400);
|
||||
}
|
||||
const caseSeeds = readAssistantSuiteCaseSeeds(runtimeCaseSetFile);
|
||||
if (caseSeeds.length === 0) {
|
||||
throw new http_1.ApiError("ASYNC_CASESET_EMPTY", "No runnable cases found in selected case-set.", 400);
|
||||
}
|
||||
const nowIso = new Date().toISOString();
|
||||
const job = {
|
||||
job_id: jobId,
|
||||
status: "queued",
|
||||
created_at: nowIso,
|
||||
updated_at: nowIso,
|
||||
eval_target: payload.evalTarget,
|
||||
run_id: runId,
|
||||
case_set_file: runtimeCaseSetFile,
|
||||
total_cases: caseSeeds.length,
|
||||
completed_cases: 0,
|
||||
cases: caseSeeds.map((item) => ({
|
||||
case_id: item.case_id,
|
||||
turns_total: item.turns_total,
|
||||
status: "queued",
|
||||
messages: []
|
||||
})),
|
||||
error: null,
|
||||
report: null
|
||||
};
|
||||
ASYNC_JOBS.set(job.job_id, job);
|
||||
trimAsyncJobsStore();
|
||||
setImmediate(() => {
|
||||
void (async () => {
|
||||
const target = ASYNC_JOBS.get(job.job_id);
|
||||
if (!target)
|
||||
return;
|
||||
target.status = "running";
|
||||
target.updated_at = new Date().toISOString();
|
||||
try {
|
||||
const report = await services.evalService.run({
|
||||
...payload,
|
||||
caseSetFile: runtimeCaseSetFile,
|
||||
runId
|
||||
});
|
||||
target.report = report;
|
||||
syncJobWithSessions(target);
|
||||
target.completed_cases = target.total_cases;
|
||||
target.status = "completed";
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
catch (error) {
|
||||
syncJobWithSessions(target);
|
||||
target.status = "failed";
|
||||
target.error = error instanceof Error ? error.message : String(error);
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
})();
|
||||
});
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
router.get("/api/eval/run-async/:job_id", (req, res, next) => {
|
||||
try {
|
||||
const jobId = String(req.params.job_id ?? "").trim();
|
||||
if (!jobId) {
|
||||
throw new http_1.ApiError("INVALID_ASYNC_JOB_ID", "job_id is required.", 400);
|
||||
}
|
||||
const job = ASYNC_JOBS.get(jobId);
|
||||
if (!job) {
|
||||
throw new http_1.ApiError("ASYNC_JOB_NOT_FOUND", `Async eval job not found: ${jobId}`, 404);
|
||||
}
|
||||
syncJobWithSessions(job);
|
||||
job.updated_at = new Date().toISOString();
|
||||
(0, http_1.ok)(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
}
|
||||
catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
return router;
|
||||
}
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ function createApp() {
|
||||
app.use((0, normalize_1.buildNormalizeRouter)(services));
|
||||
app.use((0, eval_1.buildEvalRouter)(services));
|
||||
app.use((0, assistant_1.buildAssistantRouter)(services));
|
||||
app.use((0, autoRuns_1.buildAutoRunsRouter)());
|
||||
app.use((0, autoRuns_1.buildAutoRunsRouter)(openaiClient));
|
||||
app.use((0, history_1.buildHistoryRouter)());
|
||||
app.use((0, presets_1.buildPresetsRouter)());
|
||||
app.use((0, accountingAgent_1.buildAccountingAgentRouter)(services));
|
||||
|
||||
@@ -4834,6 +4834,10 @@ class AssistantService {
|
||||
debug: null
|
||||
};
|
||||
this.sessions.appendItem(sessionId, userItem);
|
||||
const sessionAfterUserAppend = this.sessions.getSession(sessionId);
|
||||
if (sessionAfterUserAppend) {
|
||||
this.sessionLogger.persistSession(sessionAfterUserAppend);
|
||||
}
|
||||
const sessionOrganizationScope = resolveSessionOrganizationScopeContext(userMessage, session.items);
|
||||
const finalizeAddressLaneResponse = (addressLane, effectiveAddressUserMessage, carryoverMeta = null, llmPreDecomposeMeta = null) => {
|
||||
const safeAddressReply = sanitizeOutgoingAssistantText(addressLane.reply_text);
|
||||
|
||||
+8
-4
@@ -1552,7 +1552,7 @@ class EvalService {
|
||||
}
|
||||
const suite = parseAssistantSuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
|
||||
const diagnostics = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1568,6 +1568,7 @@ class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -1885,7 +1886,7 @@ class EvalService {
|
||||
}
|
||||
const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
|
||||
const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
|
||||
const diagnostics = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1903,6 +1904,7 @@ class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2177,7 +2179,8 @@ class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_stage2") {
|
||||
@@ -2187,7 +2190,8 @@ class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
if (evalTarget === "assistant_p0") {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { Router } from "express";
|
||||
import iconv from "iconv-lite";
|
||||
import {
|
||||
ASSISTANT_SESSIONS_DIR,
|
||||
AUTORUN_ANNOTATIONS_FILE,
|
||||
@@ -11,7 +12,8 @@ import {
|
||||
REPORTS_DIR
|
||||
} from "../config";
|
||||
import { ApiError, ok } from "../utils/http";
|
||||
import { loadCapabilitiesRegistry, resolveNearestCapabilityGroup } from "../services/capabilitiesRegistry";
|
||||
import { loadCapabilitiesRegistry, resolveNearestCapabilityGroup, type CapabilityGroup } from "../services/capabilitiesRegistry";
|
||||
import { OpenAIResponsesClient } from "../services/openaiResponsesClient";
|
||||
|
||||
type AutoRunTarget = "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0" | "unknown";
|
||||
type AutoRunTrend = "up" | "down" | "flat";
|
||||
@@ -144,6 +146,9 @@ interface AutoRunAnnotationRecord {
|
||||
comment: string;
|
||||
manual_case_decision: ManualCaseDecision;
|
||||
annotation_author: string | null;
|
||||
resolved: boolean;
|
||||
resolved_at: string | null;
|
||||
resolved_by: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
context: {
|
||||
@@ -154,6 +159,8 @@ interface AutoRunAnnotationRecord {
|
||||
prompt_version: string | null;
|
||||
domain: string | null;
|
||||
query_class: string | null;
|
||||
question_text: string | null;
|
||||
answer_text: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -178,9 +185,28 @@ interface AutoGenHistoryRecord {
|
||||
assistant_prompt_version: string | null;
|
||||
decomposition_prompt_version: string | null;
|
||||
prompt_fingerprint: string | null;
|
||||
autogen_personality_id: string | null;
|
||||
autogen_personality_prompt: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
interface AutoGenPersonalityCatalogItem {
|
||||
id: string;
|
||||
label: string;
|
||||
domain: string | null;
|
||||
default_prompt: string;
|
||||
source: "built_in" | "capabilities_registry";
|
||||
}
|
||||
|
||||
interface AutoGenLlmRuntimeConfig {
|
||||
llm_provider: "openai" | "local";
|
||||
api_key: string;
|
||||
model: string;
|
||||
base_url: string | null;
|
||||
temperature: number | null;
|
||||
max_output_tokens: number | null;
|
||||
}
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
@@ -254,6 +280,11 @@ function parseAnnotationAuthor(value: unknown): string | null {
|
||||
return author.slice(0, 80);
|
||||
}
|
||||
|
||||
function parseAnnotationResolved(value: unknown, fallback = false): boolean {
|
||||
const parsed = toBooleanSafe(value);
|
||||
return parsed === null ? fallback : parsed;
|
||||
}
|
||||
|
||||
function readManualDecisionSchema(): Record<string, unknown> {
|
||||
const fallback: Record<string, unknown> = {
|
||||
schema_version: "manual_case_decision_schema_v1_fallback",
|
||||
@@ -300,6 +331,8 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
|
||||
questions: toArray(item.questions)
|
||||
.map((q) => toStringSafe(q))
|
||||
.filter((q): q is string => q !== null)
|
||||
.map((q) => sanitizeGeneratedQuestion(q))
|
||||
.filter((q) => q.length > 0)
|
||||
.slice(0, 500),
|
||||
generated_by: toStringSafe(item.generated_by),
|
||||
saved_case_set_file: toStringSafe(item.saved_case_set_file),
|
||||
@@ -310,6 +343,12 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
|
||||
assistant_prompt_version: toStringSafe(toRecord(item.context)?.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(toRecord(item.context)?.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(toRecord(item.context)?.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
}))
|
||||
@@ -356,11 +395,11 @@ function collectCanonicalQuestions(limit = 300): string[] {
|
||||
for (const testCase of cases) {
|
||||
const rawQuestion = toStringSafe(testCase.raw_question) ?? toStringSafe(testCase.user_message) ?? toStringSafe(testCase.query);
|
||||
if (rawQuestion) {
|
||||
questions.push(rawQuestion);
|
||||
questions.push(sanitizeGeneratedQuestion(rawQuestion));
|
||||
}
|
||||
}
|
||||
}
|
||||
return Array.from(new Set(questions)).slice(0, limit);
|
||||
return Array.from(new Set(questions.filter((item) => item.length > 0))).slice(0, limit);
|
||||
}
|
||||
|
||||
function normalizeDomainHint(value: unknown): string | null {
|
||||
@@ -369,6 +408,55 @@ function normalizeDomainHint(value: unknown): string | null {
|
||||
return domain.toLowerCase();
|
||||
}
|
||||
|
||||
function buildAutogenPromptFromCapabilityGroup(group: CapabilityGroup): string {
|
||||
const supported = group.supported_operations.slice(0, 3).join(", ");
|
||||
const examples = group.typical_queries.slice(0, 2).join(" | ");
|
||||
const hints = group.one_c_hints.slice(0, 2).join(", ");
|
||||
const operationsPart = supported ? ` Опирайся на операции: ${supported}.` : "";
|
||||
const examplesPart = examples ? ` Ближайшие формулировки: ${examples}.` : "";
|
||||
const hintsPart = hints ? ` Можно мягко упоминать контекст 1С: ${hints}.` : "";
|
||||
return (
|
||||
`Генерируй реалистичные вопросы бухгалтера по группе "${group.group_title}".` +
|
||||
` Добавляй живую разговорную форму и опечатки, но сохраняй бизнес-смысл.${operationsPart}${examplesPart}${hintsPart}` +
|
||||
" Не выдумывай операции вне read-only режима."
|
||||
);
|
||||
}
|
||||
|
||||
function buildAutogenPersonalityCatalog(): AutoGenPersonalityCatalogItem[] {
|
||||
const builtIn: AutoGenPersonalityCatalogItem[] = [
|
||||
{
|
||||
id: "general",
|
||||
label: "Общий контур",
|
||||
domain: null,
|
||||
default_prompt:
|
||||
"Генерируй реалистичные живые вопросы бухгалтера по 1С. Добавляй разговорные формулировки и опечатки, но сохраняй бизнес-смысл.",
|
||||
source: "built_in"
|
||||
}
|
||||
];
|
||||
|
||||
const registry = loadCapabilitiesRegistry();
|
||||
const registryBased = registry.groups.map<AutoGenPersonalityCatalogItem>((group) => ({
|
||||
id: `registry_${group.group_code}`,
|
||||
label: `${group.group_title} (реестр)`,
|
||||
domain: group.group_code,
|
||||
default_prompt: buildAutogenPromptFromCapabilityGroup(group),
|
||||
source: "capabilities_registry"
|
||||
}));
|
||||
|
||||
const dedup = new Map<string, AutoGenPersonalityCatalogItem>();
|
||||
for (const item of [...builtIn, ...registryBased]) {
|
||||
if (!item.id.trim()) continue;
|
||||
if (!dedup.has(item.id)) {
|
||||
dedup.set(item.id, item);
|
||||
}
|
||||
}
|
||||
return [...dedup.values()].map((item) => ({
|
||||
...item,
|
||||
label: repairAutogenMojibake(item.label),
|
||||
default_prompt: repairAutogenMojibake(item.default_prompt)
|
||||
}));
|
||||
}
|
||||
|
||||
function fallbackDomainTemplates(domain: string | null): string[] {
|
||||
if (domain?.includes("vat") || domain?.includes("ндс")) {
|
||||
return [
|
||||
@@ -428,9 +516,9 @@ function generateQwenSeedQuestions(count: number, domain: string | null): string
|
||||
const out: string[] = [];
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = bag[index % bag.length];
|
||||
out.push(mutateIntoQwenStyle(base, index));
|
||||
out.push(sanitizeGeneratedQuestion(mutateIntoQwenStyle(base, index)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
|
||||
function generateCodexCreativeQuestions(count: number, domain: string | null): string[] {
|
||||
@@ -446,9 +534,9 @@ function generateCodexCreativeQuestions(count: number, domain: string | null): s
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const base = domainTemplates[index % domainTemplates.length];
|
||||
const pattern = patterns[index % patterns.length];
|
||||
out.push(pattern.replace("{q}", base));
|
||||
out.push(sanitizeGeneratedQuestion(pattern.replace("{q}", base)));
|
||||
}
|
||||
return Array.from(new Set(out)).slice(0, count);
|
||||
return Array.from(new Set(out.filter((item) => item.length > 0))).slice(0, count);
|
||||
}
|
||||
|
||||
function generateAutogenId(): string {
|
||||
@@ -480,6 +568,9 @@ function readAnnotations(): AutoRunAnnotationRecord[] {
|
||||
comment: toStringSafe(item.comment) ?? "",
|
||||
manual_case_decision: parseManualCaseDecision(item.manual_case_decision),
|
||||
annotation_author: parseAnnotationAuthor(item.annotation_author),
|
||||
resolved: parseAnnotationResolved(item.resolved),
|
||||
resolved_at: toStringSafe(item.resolved_at),
|
||||
resolved_by: parseAnnotationAuthor(item.resolved_by),
|
||||
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
|
||||
updated_at: toStringSafe(item.updated_at) ?? new Date().toISOString(),
|
||||
context: {
|
||||
@@ -489,7 +580,9 @@ function readAnnotations(): AutoRunAnnotationRecord[] {
|
||||
eval_target: (toStringSafe(context?.eval_target) as AutoRunTarget | null) ?? "unknown",
|
||||
prompt_version: toStringSafe(context?.prompt_version),
|
||||
domain: toStringSafe(context?.domain),
|
||||
query_class: toStringSafe(context?.query_class)
|
||||
query_class: toStringSafe(context?.query_class),
|
||||
question_text: toStringSafe(context?.question_text),
|
||||
answer_text: toStringSafe(context?.answer_text)
|
||||
}
|
||||
} satisfies AutoRunAnnotationRecord;
|
||||
})
|
||||
@@ -1156,6 +1249,51 @@ function withMessageAnnotations(
|
||||
});
|
||||
}
|
||||
|
||||
function buildRunAggregateDialog(
|
||||
run: IndexedRun,
|
||||
annotations: AutoRunAnnotationRecord[]
|
||||
): {
|
||||
source: "run_aggregate";
|
||||
session_id: string;
|
||||
messages: Array<Record<string, unknown>>;
|
||||
decomposition: string[];
|
||||
assistant_mode: Record<string, unknown> | null;
|
||||
} {
|
||||
const cases = buildCaseSummaries(run.report, run.run_id, false);
|
||||
const messages: Array<Record<string, unknown>> = [];
|
||||
const decomposition: string[] = [];
|
||||
let globalMessageIndex = 0;
|
||||
|
||||
for (const item of cases) {
|
||||
const caseId = item.case_id;
|
||||
const caseDialog = loadSessionDialog(run.run_id, caseId) ?? buildFallbackDialog(run, caseId);
|
||||
const annotatedCaseMessages = withMessageAnnotations(run.run_id, caseId, caseDialog.messages, annotations);
|
||||
|
||||
for (const caseMessage of annotatedCaseMessages) {
|
||||
const localMessageIndex = toNumberSafe(caseMessage.message_index) ?? 0;
|
||||
messages.push({
|
||||
...caseMessage,
|
||||
case_id: caseId,
|
||||
case_message_index: localMessageIndex,
|
||||
message_index: globalMessageIndex
|
||||
});
|
||||
globalMessageIndex += 1;
|
||||
}
|
||||
|
||||
if (caseDialog.decomposition.length > 0) {
|
||||
decomposition.push(...caseDialog.decomposition.map((step) => `[${caseId}] ${step}`));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
source: "run_aggregate",
|
||||
session_id: `${run.run_id}::__all__`,
|
||||
messages,
|
||||
decomposition,
|
||||
assistant_mode: null
|
||||
};
|
||||
}
|
||||
|
||||
function generateAnnotationId(): string {
|
||||
return `ann-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
||||
}
|
||||
@@ -1189,6 +1327,300 @@ function parseAutogenDomain(value: unknown): string | null {
|
||||
return domain.slice(0, 80);
|
||||
}
|
||||
|
||||
function parseAutogenLlmRuntimeConfig(
|
||||
body: Record<string, unknown>,
|
||||
context: Record<string, unknown> | null
|
||||
): AutoGenLlmRuntimeConfig | null {
|
||||
const llm = toRecord(body.llm);
|
||||
const providerRaw = toStringSafe(llm?.llm_provider ?? context?.llm_provider)?.toLowerCase() ?? "";
|
||||
const model = toStringSafe(llm?.model ?? context?.model);
|
||||
if (!model || (providerRaw !== "openai" && providerRaw !== "local")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
llm_provider: providerRaw === "local" ? "local" : "openai",
|
||||
api_key: toStringSafe(llm?.api_key) ?? "",
|
||||
model,
|
||||
base_url: toStringSafe(llm?.base_url),
|
||||
temperature: toNumberSafe(llm?.temperature),
|
||||
max_output_tokens: toNumberSafe(llm?.max_output_tokens)
|
||||
};
|
||||
}
|
||||
|
||||
function textMojibakeScore(value: string): number {
|
||||
const source = String(value ?? "");
|
||||
const cyrillic = (source.match(/[А-Яа-яЁё]/g) ?? []).length;
|
||||
const latin = (source.match(/[A-Za-z]/g) ?? []).length;
|
||||
const hardMarkers = (source.match(/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/g) ?? []).length;
|
||||
const pairMarkers = (source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length;
|
||||
const doubleEncodedMarkers = (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length;
|
||||
return cyrillic + latin - hardMarkers * 3 - pairMarkers * 2 - doubleEncodedMarkers * 2;
|
||||
}
|
||||
|
||||
function looksLikeMojibake(value: string): boolean {
|
||||
const source = String(value ?? "");
|
||||
if (!source.trim()) {
|
||||
return false;
|
||||
}
|
||||
if (/[Ѓѓ‚„…†‡€‰‹ЉЊЌЋЏ‘’“”•–—™љ›њќћџ]/.test(source)) {
|
||||
return true;
|
||||
}
|
||||
if ((source.match(/(?:Р.|С.|Ð.|Ñ.)/g) ?? []).length >= 2) {
|
||||
return true;
|
||||
}
|
||||
return (source.match(/(?:Г[Ђ-џ]|В[Ђ-џ]|Ã.|Â.)/gu) ?? []).length >= 2;
|
||||
}
|
||||
|
||||
function repairAutogenMojibake(value: string): string {
|
||||
const source = String(value ?? "");
|
||||
if (!looksLikeMojibake(source)) {
|
||||
return source;
|
||||
}
|
||||
let candidate = source;
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
let improved = false;
|
||||
try {
|
||||
const fromWin1251 = iconv.encode(candidate, "win1251").toString("utf8");
|
||||
if (textMojibakeScore(fromWin1251) > textMojibakeScore(candidate)) {
|
||||
candidate = fromWin1251;
|
||||
improved = true;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
try {
|
||||
const fromLatin1 = Buffer.from(candidate, "latin1").toString("utf8");
|
||||
if (textMojibakeScore(fromLatin1) > textMojibakeScore(candidate)) {
|
||||
candidate = fromLatin1;
|
||||
improved = true;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (!improved) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function sanitizeGeneratedQuestion(value: string): string {
|
||||
return repairAutogenMojibake(String(value ?? ""))
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitQuestionCandidates(rawText: string): string[] {
|
||||
const normalized = repairAutogenMojibake(rawText).replace(/\r/g, "\n").trim();
|
||||
if (!normalized) return [];
|
||||
|
||||
const unescaped = normalized.replace(/\\"/g, '"').replace(/\\n/g, "\n");
|
||||
const byLines = unescaped
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, ""))
|
||||
.map((line) => sanitizeGeneratedQuestion(line))
|
||||
.filter((line) => line.length > 0);
|
||||
if (byLines.length > 1) {
|
||||
return byLines;
|
||||
}
|
||||
|
||||
const questionMarkCount = (unescaped.match(/\?/g) ?? []).length;
|
||||
if (questionMarkCount > 1) {
|
||||
const byQuestion = unescaped
|
||||
.split("?")
|
||||
.map((chunk) => sanitizeGeneratedQuestion(chunk))
|
||||
.filter((chunk) => chunk.length > 0)
|
||||
.map((chunk) => (chunk.endsWith("?") ? chunk : `${chunk}?`));
|
||||
if (byQuestion.length > 1) {
|
||||
return byQuestion;
|
||||
}
|
||||
}
|
||||
|
||||
const quoted = Array.from(unescaped.matchAll(/"([^"\n]{6,}?)"/g))
|
||||
.map((match) => sanitizeGeneratedQuestion(match[1]))
|
||||
.filter((line) => line.length > 0);
|
||||
if (quoted.length > 1) {
|
||||
return quoted;
|
||||
}
|
||||
|
||||
const cleaned = sanitizeGeneratedQuestion(unescaped);
|
||||
return cleaned ? [cleaned] : [];
|
||||
}
|
||||
|
||||
function parseAutogenOutputJson(rawText: string): unknown | null {
|
||||
const cleaned = repairAutogenMojibake(rawText)
|
||||
.trim()
|
||||
.replace(/^```json\s*/i, "")
|
||||
.replace(/^```\s*/i, "")
|
||||
.replace(/```$/i, "")
|
||||
.trim();
|
||||
if (!cleaned) return null;
|
||||
try {
|
||||
return JSON.parse(cleaned) as unknown;
|
||||
} catch {
|
||||
// continue
|
||||
}
|
||||
|
||||
const arrayStart = cleaned.indexOf("[");
|
||||
const arrayEnd = cleaned.lastIndexOf("]");
|
||||
if (arrayStart >= 0 && arrayEnd > arrayStart) {
|
||||
const fragment = cleaned.slice(arrayStart, arrayEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment) as unknown;
|
||||
} catch {
|
||||
// continue
|
||||
}
|
||||
}
|
||||
|
||||
const objStart = cleaned.indexOf("{");
|
||||
const objEnd = cleaned.lastIndexOf("}");
|
||||
if (objStart >= 0 && objEnd > objStart) {
|
||||
const fragment = cleaned.slice(objStart, objEnd + 1);
|
||||
try {
|
||||
return JSON.parse(fragment) as unknown;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectQuestionsFromCandidate(value: unknown, depth = 0): string[] {
|
||||
if (depth > 5 || value === null || value === undefined) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
return value.flatMap((item) => collectQuestionsFromCandidate(item, depth + 1));
|
||||
}
|
||||
|
||||
if (typeof value === "string") {
|
||||
const text = value.trim();
|
||||
if (!text) return [];
|
||||
|
||||
const nestedParsed = parseAutogenOutputJson(text);
|
||||
if (nestedParsed !== null) {
|
||||
const nestedQuestions = collectQuestionsFromCandidate(nestedParsed, depth + 1);
|
||||
if (nestedQuestions.length > 0) {
|
||||
return nestedQuestions;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = JSON.parse(text) as unknown;
|
||||
if (decoded !== text) {
|
||||
const decodedQuestions = collectQuestionsFromCandidate(decoded, depth + 1);
|
||||
if (decodedQuestions.length > 0) {
|
||||
return decodedQuestions;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore non-JSON strings
|
||||
}
|
||||
|
||||
return splitQuestionCandidates(text);
|
||||
}
|
||||
|
||||
const record = toRecord(value);
|
||||
if (!record) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const fromQuestions = collectQuestionsFromCandidate(record.questions, depth + 1);
|
||||
if (fromQuestions.length > 0) {
|
||||
return fromQuestions;
|
||||
}
|
||||
|
||||
const fallbackText = toStringSafe(record.question ?? record.user_message ?? record.text);
|
||||
return fallbackText ? splitQuestionCandidates(fallbackText) : [];
|
||||
}
|
||||
|
||||
function extractQuestionsFromAutogenOutput(rawText: string): string[] {
|
||||
const parsed = parseAutogenOutputJson(rawText);
|
||||
const fromParsed = collectQuestionsFromCandidate(parsed);
|
||||
if (fromParsed.length > 0) {
|
||||
return fromParsed;
|
||||
}
|
||||
return collectQuestionsFromCandidate(rawText);
|
||||
}
|
||||
|
||||
async function generateQwenSeedQuestionsLive(input: {
|
||||
count: number;
|
||||
domain: string | null;
|
||||
personalityPrompt: string | null;
|
||||
llmConfig: AutoGenLlmRuntimeConfig;
|
||||
client: OpenAIResponsesClient;
|
||||
}): Promise<string[]> {
|
||||
const seedExamples = collectCanonicalQuestions(40);
|
||||
const fallbackExamples = fallbackDomainTemplates(input.domain);
|
||||
const examples = (seedExamples.length > 0 ? seedExamples : fallbackExamples).slice(0, 8);
|
||||
const personalityPrompt =
|
||||
input.personalityPrompt ??
|
||||
"Генерируй реалистичные вопросы бухгалтера по 1С. Разговорный стиль допустим, но смысл должен быть четким.";
|
||||
const repairedPersonalityPrompt = repairAutogenMojibake(personalityPrompt);
|
||||
const maxOutputTokens = clampInt(input.llmConfig.max_output_tokens, 300, 3000, 1200);
|
||||
const temperature = input.llmConfig.temperature === null ? 0.5 : Math.max(0, Math.min(1.5, input.llmConfig.temperature));
|
||||
|
||||
const systemPrompt = [
|
||||
"Ты генератор вопросов для автопрогонов бухгалтерского ассистента по 1С.",
|
||||
"Возвращай только JSON и никаких пояснений.",
|
||||
"Ассистент работает в read-only режиме: не проси действий изменения базы."
|
||||
].join(" ");
|
||||
const repairedSystemPrompt = repairAutogenMojibake(systemPrompt);
|
||||
|
||||
const developerPrompt = [
|
||||
`Нужно сгенерировать ровно ${input.count} вопросов.`,
|
||||
"Формат ответа строго:",
|
||||
'{"questions":["вопрос 1","вопрос 2"]}',
|
||||
"Требования:",
|
||||
"1) каждый вопрос отдельный, без дубликатов;",
|
||||
"2) живой пользовательский язык;",
|
||||
"3) допустимы легкие разговорные сокращения;",
|
||||
"4) не выдавай мета-комментарии и не описывай правила."
|
||||
].join("\n");
|
||||
const repairedDeveloperPrompt = repairAutogenMojibake(developerPrompt);
|
||||
|
||||
const userMessage = [
|
||||
`Домен: ${input.domain ?? "general"}.`,
|
||||
`Промпт личности: ${repairedPersonalityPrompt}`,
|
||||
"Примеры ориентиров по стилю и тематике:",
|
||||
...examples.map((item, index) => `${index + 1}. ${item}`)
|
||||
].join("\n");
|
||||
const repairedUserMessage = repairAutogenMojibake(userMessage);
|
||||
|
||||
const response = await input.client.chat(
|
||||
{
|
||||
llmProvider: input.llmConfig.llm_provider,
|
||||
apiKey: input.llmConfig.api_key,
|
||||
model: input.llmConfig.model,
|
||||
baseUrl: input.llmConfig.base_url ?? undefined,
|
||||
temperature,
|
||||
maxOutputTokens: maxOutputTokens
|
||||
},
|
||||
{
|
||||
systemPrompt: repairedSystemPrompt,
|
||||
developerPrompt: repairedDeveloperPrompt,
|
||||
userMessage: repairedUserMessage,
|
||||
temperature,
|
||||
maxOutputTokens
|
||||
}
|
||||
);
|
||||
|
||||
const extracted = extractQuestionsFromAutogenOutput(response.outputText);
|
||||
const normalized = Array.from(new Set(extracted.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
|
||||
if (normalized.length === 0) {
|
||||
throw new ApiError("AUTOGEN_LLM_EMPTY_OUTPUT", "Qwen не вернул пригодные вопросы для автогенерации.", 502, {
|
||||
model: input.llmConfig.model
|
||||
});
|
||||
}
|
||||
|
||||
const fallback = generateQwenSeedQuestions(input.count, input.domain);
|
||||
return Array.from(new Set([...normalized, ...fallback])).slice(0, input.count);
|
||||
}
|
||||
|
||||
function hasAnyRunFilterQuery(query: Record<string, unknown>): boolean {
|
||||
return Boolean(
|
||||
toStringSafe(query.from) ??
|
||||
@@ -1219,7 +1651,10 @@ function buildAutogenCaseSetPayload(input: {
|
||||
domain: string | null;
|
||||
questions: string[];
|
||||
}): Record<string, unknown> {
|
||||
const cases = input.questions.map((question, index) => ({
|
||||
const normalizedQuestions = Array.from(
|
||||
new Set(input.questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))
|
||||
);
|
||||
const cases = normalizedQuestions.map((question, index) => ({
|
||||
case_id: `AUTO-${String(index + 1).padStart(3, "0")}`,
|
||||
scenario_tag: `${input.mode}_${input.domain ?? "general"}`,
|
||||
question_type: "direct",
|
||||
@@ -1341,7 +1776,7 @@ function collectPostAnalysis(
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAutoRunsRouter(): Router {
|
||||
export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()): Router {
|
||||
const router = Router();
|
||||
|
||||
router.get("/api/autoruns/history", (req, res) => {
|
||||
@@ -1423,9 +1858,23 @@ export function buildAutoRunsRouter(): Router {
|
||||
throw new ApiError("AUTORUN_NOT_FOUND", `Run not found: ${runId}`, 404);
|
||||
}
|
||||
|
||||
const annotations = readAnnotations();
|
||||
if (caseId === "__all__") {
|
||||
const dialog = buildRunAggregateDialog(run, annotations);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
run_id: runId,
|
||||
case_id: "__all__",
|
||||
...dialog,
|
||||
annotations: annotations
|
||||
.filter((item) => item.run_id === runId)
|
||||
.sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at))
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionDialog = loadSessionDialog(runId, caseId);
|
||||
const dialog = sessionDialog ?? buildFallbackDialog(run, caseId);
|
||||
const annotations = readAnnotations();
|
||||
const messages = withMessageAnnotations(runId, caseId, dialog.messages, annotations);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
@@ -1563,6 +2012,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
if (targetRole !== "assistant") {
|
||||
throw new ApiError("AUTORUN_MESSAGE_NOT_ASSISTANT", "Only assistant answers can be annotated", 400);
|
||||
}
|
||||
const pairedUserQuestion = [...dialog.messages.slice(0, messageIndex)]
|
||||
.reverse()
|
||||
.find((item) => (toStringSafe(item.role) ?? "") === "user");
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const annotations = readAnnotations();
|
||||
@@ -1580,6 +2032,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
comment,
|
||||
manual_case_decision: manualCaseDecision,
|
||||
annotation_author: annotationAuthor,
|
||||
resolved: existing?.resolved ?? false,
|
||||
resolved_at: existing?.resolved_at ?? null,
|
||||
resolved_by: existing?.resolved_by ?? null,
|
||||
created_at: existing?.created_at ?? nowIso,
|
||||
updated_at: nowIso,
|
||||
context: {
|
||||
@@ -1589,7 +2044,9 @@ export function buildAutoRunsRouter(): Router {
|
||||
eval_target: run.eval_target,
|
||||
prompt_version: toStringSafe(run.report.prompt_version),
|
||||
domain: caseSummary.domain,
|
||||
query_class: caseSummary.query_class
|
||||
query_class: caseSummary.query_class,
|
||||
question_text: toStringSafe(pairedUserQuestion?.text),
|
||||
answer_text: toStringSafe(targetMessage.text)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1613,6 +2070,56 @@ export function buildAutoRunsRouter(): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.patch("/api/autoruns/annotations/:annotation_id", (req, res, next) => {
|
||||
try {
|
||||
const annotationId = toStringSafe(req.params.annotation_id);
|
||||
if (!annotationId) {
|
||||
throw new ApiError("INVALID_ANNOTATION_ID", "annotation_id is required", 400);
|
||||
}
|
||||
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
throw new ApiError("INVALID_ANNOTATION_PATCH", "JSON body is required", 400);
|
||||
}
|
||||
|
||||
const resolved = toBooleanSafe(body.resolved);
|
||||
if (resolved === null) {
|
||||
throw new ApiError("INVALID_ANNOTATION_PATCH", "resolved flag is required", 400);
|
||||
}
|
||||
const resolvedBy = parseAnnotationAuthor(body.resolved_by);
|
||||
|
||||
const annotations = readAnnotations();
|
||||
const index = annotations.findIndex((item) => item.annotation_id === annotationId);
|
||||
if (index < 0) {
|
||||
throw new ApiError("ANNOTATION_NOT_FOUND", `Annotation not found: ${annotationId}`, 404);
|
||||
}
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const current = annotations[index];
|
||||
const updated: AutoRunAnnotationRecord = {
|
||||
...current,
|
||||
resolved,
|
||||
resolved_at: resolved ? nowIso : null,
|
||||
resolved_by: resolved ? resolvedBy ?? current.resolved_by ?? null : null,
|
||||
updated_at: nowIso
|
||||
};
|
||||
|
||||
annotations[index] = updated;
|
||||
writeAnnotations(annotations);
|
||||
|
||||
const statsByCase = buildAnnotationStatsMap(updated.run_id, annotations);
|
||||
const caseStats = statsByCase.get(updated.case_id) ?? null;
|
||||
|
||||
ok(res, {
|
||||
ok: true,
|
||||
annotation: updated,
|
||||
case_annotation_stats: caseStats
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/api/autoruns/manual-decision-schema", (_req, res) => {
|
||||
ok(res, {
|
||||
ok: true,
|
||||
@@ -1682,7 +2189,19 @@ export function buildAutoRunsRouter(): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/autoruns/autogen/generate", (req, res, next) => {
|
||||
router.get("/api/autoruns/autogen/personality-catalog", (_req, res, next) => {
|
||||
try {
|
||||
ok(res, {
|
||||
ok: true,
|
||||
generated_at: new Date().toISOString(),
|
||||
items: buildAutogenPersonalityCatalog()
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
|
||||
try {
|
||||
const body = toRecord(req.body);
|
||||
if (!body) {
|
||||
@@ -1694,11 +2213,32 @@ export function buildAutoRunsRouter(): Router {
|
||||
const persistCaseSet = toBooleanSafe(body.persist_to_eval_cases) ?? true;
|
||||
const generatedBy = parseAnnotationAuthor(body.generated_by);
|
||||
const context = toRecord(body.context);
|
||||
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
|
||||
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
|
||||
|
||||
const questions =
|
||||
mode === "qwen_seed"
|
||||
? generateQwenSeedQuestions(count, domain)
|
||||
: generateCodexCreativeQuestions(count, domain);
|
||||
let questions: string[] = [];
|
||||
if (mode === "qwen_seed") {
|
||||
if (!llmConfig) {
|
||||
throw new ApiError(
|
||||
"AUTOGEN_LLM_CONFIG_REQUIRED",
|
||||
"Для режима qwen_seed нужен активный LLM-контур (provider/model/baseUrl) из настроек подключения.",
|
||||
400
|
||||
);
|
||||
}
|
||||
questions = await generateQwenSeedQuestionsLive({
|
||||
count,
|
||||
domain,
|
||||
personalityPrompt,
|
||||
llmConfig,
|
||||
client: openaiClient
|
||||
});
|
||||
} else {
|
||||
questions = generateCodexCreativeQuestions(count, domain);
|
||||
}
|
||||
questions = Array.from(new Set(questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0))).slice(
|
||||
0,
|
||||
count
|
||||
);
|
||||
const generationId = generateAutogenId();
|
||||
|
||||
let savedCaseSetFile: string | null = null;
|
||||
@@ -1734,6 +2274,12 @@ export function buildAutoRunsRouter(): Router {
|
||||
assistant_prompt_version: toStringSafe(context.assistant_prompt_version),
|
||||
decomposition_prompt_version: toStringSafe(context.decomposition_prompt_version),
|
||||
prompt_fingerprint: toStringSafe(context.prompt_fingerprint)
|
||||
? repairAutogenMojibake(String(context.prompt_fingerprint))
|
||||
: null,
|
||||
autogen_personality_id: toStringSafe(context.autogen_personality_id),
|
||||
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
|
||||
? repairAutogenMojibake(String(context.autogen_personality_prompt))
|
||||
: null
|
||||
}
|
||||
: null
|
||||
};
|
||||
@@ -1752,3 +2298,5 @@ export function buildAutoRunsRouter(): Router {
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,30 +1,333 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { nanoid } from "nanoid";
|
||||
import { Router } from "express";
|
||||
import { ASSISTANT_SESSIONS_DIR, EVAL_CASES_DIR, EVAL_DATASETS_DIR } from "../config";
|
||||
import type { AppServices } from "../serverContext";
|
||||
import { ok } from "../utils/http";
|
||||
import { ApiError, ok } from "../utils/http";
|
||||
import type { EvalRunMode, NormalizeRequestPayload } from "../types/normalizer";
|
||||
import type { EvalTarget } from "../types/assistantEval";
|
||||
|
||||
type EvalAsyncStatus = "queued" | "running" | "completed" | "failed";
|
||||
|
||||
interface EvalAsyncCaseInfo {
|
||||
case_id: string;
|
||||
turns_total: number;
|
||||
status: EvalAsyncStatus;
|
||||
messages: Array<{
|
||||
message_id: string | null;
|
||||
role: string;
|
||||
text: string;
|
||||
created_at: string | null;
|
||||
trace_id: string | null;
|
||||
reply_type: string | null;
|
||||
message_index: number;
|
||||
case_id: string;
|
||||
case_message_index: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface EvalAsyncJob {
|
||||
job_id: string;
|
||||
status: EvalAsyncStatus;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
eval_target: EvalTarget;
|
||||
run_id: string;
|
||||
case_set_file: string | null;
|
||||
total_cases: number;
|
||||
completed_cases: number;
|
||||
cases: EvalAsyncCaseInfo[];
|
||||
error: string | null;
|
||||
report: Record<string, unknown> | null;
|
||||
}
|
||||
|
||||
const ASYNC_JOBS = new Map<string, EvalAsyncJob>();
|
||||
const MAX_ASYNC_JOBS = 80;
|
||||
|
||||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function toStringSafe(value: unknown): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : null;
|
||||
}
|
||||
|
||||
function toArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function normalizeQuestionChunk(value: string): string {
|
||||
return String(value ?? "")
|
||||
.replace(/\r/g, " ")
|
||||
.replace(/\t/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function splitQuestionCandidate(raw: string): string[] {
|
||||
const normalized = String(raw ?? "").replace(/\r/g, "\n").trim();
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const byLines = normalized
|
||||
.split(/\n+/g)
|
||||
.map((line) => line.replace(/^\s*(?:[-*•]|\d{1,3}[).:]?)\s*/, "").trim())
|
||||
.filter((line) => line.length > 0);
|
||||
const source = byLines.length > 1 ? byLines : [normalized];
|
||||
|
||||
const chunks: string[] = [];
|
||||
for (const line of source) {
|
||||
const questionLike = Array.from(line.matchAll(/[^?]+(?:\?|$)/g))
|
||||
.map((match) => normalizeQuestionChunk(match[0]))
|
||||
.filter((item) => item.length > 0);
|
||||
if (questionLike.length > 1) {
|
||||
for (const item of questionLike) {
|
||||
chunks.push(item.endsWith("?") ? item : `${item}?`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chunks.push(normalizeQuestionChunk(line));
|
||||
}
|
||||
return chunks.filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function normalizeRuntimeQuestions(value: unknown): string[] {
|
||||
const raw = toArray(value)
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
if (raw.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const expanded = raw.flatMap((item) => splitQuestionCandidate(item));
|
||||
const deduped: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const item of expanded) {
|
||||
const normalized = normalizeQuestionChunk(item);
|
||||
if (!normalized) continue;
|
||||
if (seen.has(normalized)) continue;
|
||||
seen.add(normalized);
|
||||
deduped.push(normalized);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
function normalizeCaseIds(value: unknown): string[] | undefined {
|
||||
if (!Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = value
|
||||
.map((item) => (typeof item === "string" ? item.trim() : ""))
|
||||
.filter((item) => item.length > 0);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function buildEvalPayloadFromBody(body: Record<string, unknown>): {
|
||||
normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
|
||||
caseIds?: string[];
|
||||
useMock: boolean;
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
rawQuestions?: string;
|
||||
evalTarget: EvalTarget;
|
||||
compareWithReportFile?: string;
|
||||
} {
|
||||
return {
|
||||
normalizeConfig: (body.normalizeConfig ?? {}) as Omit<NormalizeRequestPayload, "userQuestion" | "context">,
|
||||
caseIds: normalizeCaseIds(body.caseIds),
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: (body.mode as EvalRunMode | undefined) ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: (body.eval_target as EvalTarget | undefined) ?? "normalizer",
|
||||
compareWithReportFile:
|
||||
typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
};
|
||||
}
|
||||
|
||||
function resolveReadablePath(inputPath: string): string {
|
||||
if (path.isAbsolute(inputPath)) {
|
||||
return inputPath;
|
||||
}
|
||||
const candidates = [
|
||||
path.resolve(EVAL_CASES_DIR, inputPath),
|
||||
path.resolve(EVAL_DATASETS_DIR, inputPath),
|
||||
path.resolve(inputPath)
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
function readAssistantSuiteCaseSeeds(inputPath: string): Array<{ case_id: string; turns_total: number }> {
|
||||
const filePath = resolveReadablePath(inputPath);
|
||||
const raw = fs.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
const record = toRecord(parsed);
|
||||
const cases = toArray(record?.cases);
|
||||
return cases
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item): item is Record<string, unknown> => item !== null)
|
||||
.map((item) => {
|
||||
const caseId = toStringSafe(item.case_id);
|
||||
const turns = toArray(item.turns);
|
||||
if (!caseId || turns.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
case_id: caseId,
|
||||
turns_total: turns.length
|
||||
};
|
||||
})
|
||||
.filter((item): item is { case_id: string; turns_total: number } => item !== null);
|
||||
}
|
||||
|
||||
function writeRuntimeAssistantSuiteFromQuestions(jobId: string, questions: string[]): string {
|
||||
if (!fs.existsSync(EVAL_CASES_DIR)) {
|
||||
fs.mkdirSync(EVAL_CASES_DIR, { recursive: true });
|
||||
}
|
||||
const cases = questions.map((question, index) => {
|
||||
const caseId = `AUTO-${String(index + 1).padStart(3, "0")}`;
|
||||
return {
|
||||
case_id: caseId,
|
||||
scenario_tag: "autogen_runtime",
|
||||
question_type: "direct",
|
||||
broadness_level: "medium",
|
||||
turns: [{ user_message: question }]
|
||||
};
|
||||
});
|
||||
const payload = {
|
||||
suite_id: `assistant_autogen_runtime_${jobId}`,
|
||||
suite_version: "0.1.0",
|
||||
schema_version: "assistant_autogen_runtime_v0_1",
|
||||
scenario_count: cases.length,
|
||||
case_ids: cases.map((item) => item.case_id),
|
||||
cases
|
||||
};
|
||||
const fileName = `assistant_autogen_runtime_${jobId}.json`;
|
||||
fs.writeFileSync(path.resolve(EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
|
||||
return fileName;
|
||||
}
|
||||
|
||||
function readSessionConversation(runId: string, caseId: string): EvalAsyncCaseInfo["messages"] {
|
||||
const sessionId = `${runId}-${caseId}`;
|
||||
const filePath = path.resolve(ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8")) as unknown;
|
||||
const record = toRecord(parsed);
|
||||
const conversation = toArray(record?.conversation)
|
||||
.map((item) => toRecord(item))
|
||||
.filter((item): item is Record<string, unknown> => item !== null);
|
||||
return conversation.map((item, index) => ({
|
||||
message_id: toStringSafe(item.message_id),
|
||||
role: toStringSafe(item.role) ?? "unknown",
|
||||
text: toStringSafe(item.text) ?? "",
|
||||
created_at: toStringSafe(item.created_at),
|
||||
trace_id: toStringSafe(item.trace_id),
|
||||
reply_type: toStringSafe(item.reply_type),
|
||||
message_index: index,
|
||||
case_id: caseId,
|
||||
case_message_index: index
|
||||
}));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function syncJobWithSessions(job: EvalAsyncJob): void {
|
||||
if (!job.run_id || !job.eval_target.startsWith("assistant_")) {
|
||||
return;
|
||||
}
|
||||
let completed = 0;
|
||||
let hasRunning = false;
|
||||
for (const item of job.cases) {
|
||||
const messages = readSessionConversation(job.run_id, item.case_id);
|
||||
item.messages = messages;
|
||||
const assistantMessages = messages.filter((entry) => entry.role === "assistant").length;
|
||||
const userMessages = messages.filter((entry) => entry.role === "user").length;
|
||||
if (assistantMessages >= item.turns_total && item.turns_total > 0) {
|
||||
item.status = "completed";
|
||||
completed += 1;
|
||||
continue;
|
||||
}
|
||||
if (userMessages > 0 || messages.length > 0) {
|
||||
item.status = "running";
|
||||
hasRunning = true;
|
||||
continue;
|
||||
}
|
||||
item.status = "queued";
|
||||
}
|
||||
job.completed_cases = completed;
|
||||
if (job.status === "running" && !hasRunning && completed === job.total_cases && job.total_cases > 0) {
|
||||
job.status = "completed";
|
||||
}
|
||||
}
|
||||
|
||||
function trimAsyncJobsStore(): void {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS) return;
|
||||
const sorted = Array.from(ASYNC_JOBS.values()).sort((a, b) => Date.parse(a.updated_at) - Date.parse(b.updated_at));
|
||||
for (const item of sorted) {
|
||||
if (ASYNC_JOBS.size <= MAX_ASYNC_JOBS) break;
|
||||
ASYNC_JOBS.delete(item.job_id);
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotJob(job: EvalAsyncJob): Record<string, unknown> {
|
||||
return {
|
||||
job_id: job.job_id,
|
||||
status: job.status,
|
||||
created_at: job.created_at,
|
||||
updated_at: job.updated_at,
|
||||
eval_target: job.eval_target,
|
||||
run_id: job.run_id,
|
||||
case_set_file: job.case_set_file,
|
||||
total_cases: job.total_cases,
|
||||
completed_cases: job.completed_cases,
|
||||
error: job.error,
|
||||
cases: job.cases,
|
||||
report_summary: job.report
|
||||
? {
|
||||
run_id: toStringSafe(job.report.run_id),
|
||||
run_timestamp: toStringSafe(job.report.run_timestamp) ?? toStringSafe(job.report.timestamp),
|
||||
score_index:
|
||||
typeof job.report.score_index === "number"
|
||||
? Number(job.report.score_index)
|
||||
: toRecord(job.report.metrics) && typeof toRecord(job.report.metrics)?.score_index === "number"
|
||||
? Number(toRecord(job.report.metrics)?.score_index)
|
||||
: null,
|
||||
cases_total: typeof job.report.cases_total === "number" ? Number(job.report.cases_total) : null
|
||||
}
|
||||
: null
|
||||
};
|
||||
}
|
||||
|
||||
export function buildEvalRouter(services: AppServices): Router {
|
||||
const router = Router();
|
||||
|
||||
router.post("/api/eval/run", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {}) as Record<string, unknown>;
|
||||
const report = await services.evalService.run({
|
||||
normalizeConfig: (body.normalizeConfig ?? {}) as Omit<NormalizeRequestPayload, "userQuestion" | "context">,
|
||||
caseIds: Array.isArray(body.caseIds) ? (body.caseIds as string[]) : undefined,
|
||||
useMock: Boolean(body.useMock),
|
||||
mode: (body.mode as EvalRunMode | undefined) ?? "standard",
|
||||
caseSetFile: typeof body.caseSetFile === "string" ? body.caseSetFile : undefined,
|
||||
rawQuestions: typeof body.rawQuestions === "string" ? body.rawQuestions : undefined,
|
||||
evalTarget: (body.eval_target as EvalTarget | undefined) ?? "normalizer",
|
||||
compareWithReportFile:
|
||||
typeof body.compare_with_report_file === "string"
|
||||
? body.compare_with_report_file
|
||||
: typeof body.comparisonBaselineReportFile === "string"
|
||||
? body.comparisonBaselineReportFile
|
||||
: undefined
|
||||
});
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
const report = await services.evalService.run(payload);
|
||||
ok(res, {
|
||||
ok: true,
|
||||
report
|
||||
@@ -34,5 +337,115 @@ export function buildEvalRouter(services: AppServices): Router {
|
||||
}
|
||||
});
|
||||
|
||||
router.post("/api/eval/run-async/start", async (req, res, next) => {
|
||||
try {
|
||||
const body = (req.body ?? {}) as Record<string, unknown>;
|
||||
const payload = buildEvalPayloadFromBody(body);
|
||||
if (payload.evalTarget !== "assistant_stage1") {
|
||||
throw new ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
|
||||
}
|
||||
const questions = normalizeRuntimeQuestions(body.questions);
|
||||
|
||||
const jobId = `job-${nanoid(10)}`;
|
||||
const runId = `assistant-stage1-${nanoid(10)}`;
|
||||
const runtimeCaseSetFile =
|
||||
questions.length > 0
|
||||
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
|
||||
: payload.caseSetFile
|
||||
? payload.caseSetFile
|
||||
: undefined;
|
||||
|
||||
if (!runtimeCaseSetFile) {
|
||||
throw new ApiError(
|
||||
"ASYNC_CASESET_REQUIRED",
|
||||
"Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.",
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
const caseSeeds = readAssistantSuiteCaseSeeds(runtimeCaseSetFile);
|
||||
if (caseSeeds.length === 0) {
|
||||
throw new ApiError("ASYNC_CASESET_EMPTY", "No runnable cases found in selected case-set.", 400);
|
||||
}
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const job: EvalAsyncJob = {
|
||||
job_id: jobId,
|
||||
status: "queued",
|
||||
created_at: nowIso,
|
||||
updated_at: nowIso,
|
||||
eval_target: payload.evalTarget,
|
||||
run_id: runId,
|
||||
case_set_file: runtimeCaseSetFile,
|
||||
total_cases: caseSeeds.length,
|
||||
completed_cases: 0,
|
||||
cases: caseSeeds.map((item) => ({
|
||||
case_id: item.case_id,
|
||||
turns_total: item.turns_total,
|
||||
status: "queued",
|
||||
messages: []
|
||||
})),
|
||||
error: null,
|
||||
report: null
|
||||
};
|
||||
ASYNC_JOBS.set(job.job_id, job);
|
||||
trimAsyncJobsStore();
|
||||
|
||||
setImmediate(() => {
|
||||
void (async () => {
|
||||
const target = ASYNC_JOBS.get(job.job_id);
|
||||
if (!target) return;
|
||||
target.status = "running";
|
||||
target.updated_at = new Date().toISOString();
|
||||
try {
|
||||
const report = await services.evalService.run({
|
||||
...payload,
|
||||
caseSetFile: runtimeCaseSetFile,
|
||||
runId
|
||||
});
|
||||
target.report = report;
|
||||
syncJobWithSessions(target);
|
||||
target.completed_cases = target.total_cases;
|
||||
target.status = "completed";
|
||||
target.updated_at = new Date().toISOString();
|
||||
} catch (error) {
|
||||
syncJobWithSessions(target);
|
||||
target.status = "failed";
|
||||
target.error = error instanceof Error ? error.message : String(error);
|
||||
target.updated_at = new Date().toISOString();
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
ok(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
router.get("/api/eval/run-async/:job_id", (req, res, next) => {
|
||||
try {
|
||||
const jobId = String(req.params.job_id ?? "").trim();
|
||||
if (!jobId) {
|
||||
throw new ApiError("INVALID_ASYNC_JOB_ID", "job_id is required.", 400);
|
||||
}
|
||||
const job = ASYNC_JOBS.get(jobId);
|
||||
if (!job) {
|
||||
throw new ApiError("ASYNC_JOB_NOT_FOUND", `Async eval job not found: ${jobId}`, 404);
|
||||
}
|
||||
syncJobWithSessions(job);
|
||||
job.updated_at = new Date().toISOString();
|
||||
ok(res, {
|
||||
ok: true,
|
||||
job: snapshotJob(job)
|
||||
});
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ export function createApp(): express.Express {
|
||||
app.use(buildNormalizeRouter(services));
|
||||
app.use(buildEvalRouter(services));
|
||||
app.use(buildAssistantRouter(services));
|
||||
app.use(buildAutoRunsRouter());
|
||||
app.use(buildAutoRunsRouter(openaiClient));
|
||||
app.use(buildHistoryRouter());
|
||||
app.use(buildPresetsRouter());
|
||||
app.use(buildAccountingAgentRouter(services));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// @ts-nocheck
|
||||
// @ts-nocheck
|
||||
import * as nanoid_1 from "nanoid";
|
||||
import * as stage1Contracts_1 from "../types/stage1Contracts";
|
||||
import * as config_1 from "../config";
|
||||
@@ -4789,6 +4789,10 @@ export class AssistantService {
|
||||
debug: null
|
||||
};
|
||||
this.sessions.appendItem(sessionId, userItem);
|
||||
const sessionAfterUserAppend = this.sessions.getSession(sessionId);
|
||||
if (sessionAfterUserAppend) {
|
||||
this.sessionLogger.persistSession(sessionAfterUserAppend);
|
||||
}
|
||||
const sessionOrganizationScope = resolveSessionOrganizationScopeContext(userMessage, session.items);
|
||||
const finalizeAddressLaneResponse = (addressLane, effectiveAddressUserMessage, carryoverMeta = null, llmPreDecomposeMeta = null) => {
|
||||
const safeAddressReply = sanitizeOutgoingAssistantText(addressLane.reply_text);
|
||||
|
||||
@@ -1876,6 +1876,7 @@ export class EvalService {
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
@@ -1887,7 +1888,7 @@ export class EvalService {
|
||||
|
||||
const suite = parseAssistantSuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage1-${nanoid(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage1-${nanoid(10)}`;
|
||||
const assistantService = new AssistantService(this.normalizerService, new AssistantSessionStore());
|
||||
const diagnostics: AssistantCaseDiagnostics[] = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -1905,6 +1906,7 @@ export class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2223,6 +2225,7 @@ export class EvalService {
|
||||
mode: EvalRunMode;
|
||||
caseSetFile?: string;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
if (!FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
|
||||
throw new ApiError(
|
||||
@@ -2234,7 +2237,7 @@ export class EvalService {
|
||||
|
||||
const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
|
||||
const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
|
||||
const runId = `assistant-stage2-${nanoid(10)}`;
|
||||
const runId = typeof payload.runId === "string" && payload.runId.trim().length > 0 ? payload.runId.trim() : `assistant-stage2-${nanoid(10)}`;
|
||||
const assistantService = new AssistantService(this.normalizerService, new AssistantSessionStore());
|
||||
const diagnostics: AssistantStage2CaseDiagnostics[] = [];
|
||||
let requestsTotal = 0;
|
||||
@@ -2255,6 +2258,7 @@ export class EvalService {
|
||||
user_message: turn.user_message,
|
||||
message: turn.user_message,
|
||||
mode: "assistant",
|
||||
llmProvider: payload.normalizeConfig.llmProvider,
|
||||
apiKey: payload.normalizeConfig.apiKey,
|
||||
model: payload.normalizeConfig.model,
|
||||
baseUrl: payload.normalizeConfig.baseUrl,
|
||||
@@ -2548,6 +2552,7 @@ export class EvalService {
|
||||
rawQuestions?: string;
|
||||
evalTarget?: EvalTarget;
|
||||
compareWithReportFile?: string;
|
||||
runId?: string;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const mode = payload.mode ?? "standard";
|
||||
const evalTarget = payload.evalTarget ?? "normalizer";
|
||||
@@ -2559,7 +2564,8 @@ export class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2570,7 +2576,8 @@ export class EvalService {
|
||||
useMock: payload.useMock,
|
||||
mode,
|
||||
caseSetFile: payload.caseSetFile,
|
||||
compareWithReportFile: payload.compareWithReportFile
|
||||
compareWithReportFile: payload.compareWithReportFile,
|
||||
runId: payload.runId
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
[
|
||||
{
|
||||
"annotation_id": "ann-mnrol1zq-6eiif5d",
|
||||
"run_id": "assistant-stage1-5513CzxPo1",
|
||||
"case_id": "AUTO-001",
|
||||
"session_id": "assistant-stage1-5513CzxPo1-AUTO-001",
|
||||
"message_index": 1,
|
||||
"rating": 1,
|
||||
"comment": "Тут брак не только в формулировке ответа, а в самой логике попадания в запрос:\n\nвопрос про поставщиков с хвостами на конец месяца;\nещё и про признак систематической проблемы, а не разовую задержку;\nв ответе показан один документ списания с расчётного счёта по одному контрагенту;\nне объяснено, почему этот документ вообще релевантен запросу;\nне раскрыто, как он связан с хвостами, концом месяца и системностью;\nнет декомпозиции запроса и нет явного ответа на его основную часть.\n\nТо есть система фактически подменила задачу: вместо поиска паттерна по поставщикам выдала одиночную находку без обоснования.\n\n\n\nОтвет не попал в суть запроса: вопрос был о поставщиках с системными хвостами на конец месяца, а показан одиночный документ списания без объяснения его связи с хвостами, периодом и систематичностью. Нужны декомпозиция запроса, явное обоснование выбора документа и комментарий о степени покрытия контекста.",
|
||||
"manual_case_decision": "needs_dialog_policy_fix",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T16:18:03.734Z",
|
||||
"updated_at": "2026-04-09T16:18:03.734Z",
|
||||
"context": {
|
||||
"message_id": "msg-hHz2ydMJOa",
|
||||
"trace_id": "address-z5C0-D-mMH",
|
||||
"reply_type": "factual",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "Кто из поставщиков имеет хвосты с документами на конец месяца, которые уже больше похожи на систематическую проблему, а не на обычную задержку?",
|
||||
"answer_text": "Найдено документов по контрагенту: 1.\n1. 2020-11-16T16:08:51Z | Списание с расчетного счета 00000000262 от 16.11.2020 16:08:51 | 0 / 0 | 8700 | аналитика: ВИЗАНТИЯ; Счет № 3363 от 16.11.2020 г."
|
||||
}
|
||||
},
|
||||
{
|
||||
"annotation_id": "ann-mnrsn1s4-g1t5tks",
|
||||
"run_id": "assistant-stage1-rDM8XCbg7q",
|
||||
"case_id": "AUTO-001",
|
||||
"session_id": "assistant-stage1-rDM8XCbg7q-AUTO-001",
|
||||
"message_index": 1,
|
||||
"rating": 3,
|
||||
"comment": "Так, ну этот вопрос явно вне контура, тут ответ очень технический. Ну как бы юзеру вообще не надо знать ничего про адрес Query, это какая-то техническая хуета, ее нахуй не нужно вываливать. Вот. И просить переформулировать тоже не надо, потому что когда мы просим что-то переформулировать, это значит, мы что-то не поняли. Тут явно вопрос вне контура. Вот. Ну то есть мы как бы точно не будем разлиновку делать под этот запрос.",
|
||||
"manual_case_decision": "out_of_scope_but_answer_softly",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T18:11:35.236Z",
|
||||
"updated_at": "2026-04-09T18:11:35.236Z",
|
||||
"context": {
|
||||
"message_id": "msg-JqLbA9uuLs",
|
||||
"trace_id": "address-vyNV6r-l6T",
|
||||
"reply_type": "partial_coverage",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "Какие поставщики у нас уже пару месяцев сдают акты без приходок. Может, их надо проконтролировать отдельно чтоб не засорять бухгалтерию дальше?",
|
||||
"answer_text": "Этот запрос не подходит под address_query V1.\nПричина: intent пока не поддержан в address V1.\nЧто нужно уточнить: переформулируйте вопрос как адресный lookup по счету/контрагенту/договору."
|
||||
}
|
||||
},
|
||||
{
|
||||
"annotation_id": "ann-mnrsrgxc-tstn1f1",
|
||||
"run_id": "assistant-stage1-rDM8XCbg7q",
|
||||
"case_id": "AUTO-002",
|
||||
"session_id": "assistant-stage1-rDM8XCbg7q-AUTO-002",
|
||||
"message_index": 1,
|
||||
"rating": 3,
|
||||
"comment": "Так, ну смотри, короче, ответ очень опять очень технический, не хватает обязательного якоря. Тут как бы мы не должны ни про какие там якоря там обязательно говорить, что есть обязательный якорь. Ну как бы это знаешь, такая очень абстрактная история. Так, дальше ответ. Причина контрагент по указанному имени Алясу не найден. Да только кто-то вообще не уточняет ни про каких конкретных. Он как раз и требует тебе найти какого-то контрагента. Он не знает. Он вот тут вопрос как раз про поиски. Вот, и точно- точно неправильная просьба. Тут надо смотреть на диапазоны, наверное, между заключением договора и там какими-то оплатами. Вот, и предполагать, что это оно как раз. То, что очень длинные окна, там они могут на годы там быть, например, я не знаю. Ну, короче, сыро очень ответчик. Очень сыро ответил и вообще не по теме.",
|
||||
"manual_case_decision": "covered_but_bad_answer",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T18:15:01.488Z",
|
||||
"updated_at": "2026-04-09T18:15:01.488Z",
|
||||
"context": {
|
||||
"message_id": "msg-ONDlFjoxbU",
|
||||
"trace_id": "address-paNbrPvWpC",
|
||||
"reply_type": "partial_coverage",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "Где у нас висят покупатели со слишком длинным периодом между отправкой товара и его оплатой, и это уже вызывает тревогу?",
|
||||
"answer_text": "Для точного адресного поиска не хватает обязательного якоря.\nПричина: контрагент по указанному имени/алиасу не найден в materialized live-строках.\nЧто нужно уточнить: уточните точное имя контрагента или добавьте ИНН."
|
||||
}
|
||||
},
|
||||
{
|
||||
"annotation_id": "ann-mnrstmmx-dqr0qoq",
|
||||
"run_id": "assistant-stage1-rDM8XCbg7q",
|
||||
"case_id": "AUTO-003",
|
||||
"session_id": "assistant-stage1-rDM8XCbg7q-AUTO-003",
|
||||
"message_index": 1,
|
||||
"rating": 1,
|
||||
"comment": "Так мы вообще не должны вводить вот эту техническую историю про что сломано. Вот, я, если честно, скорее всего, надо просто ответить, что вопрос вне контекста.",
|
||||
"manual_case_decision": "out_of_scope_but_answer_softly",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T18:16:42.200Z",
|
||||
"updated_at": "2026-04-09T18:16:42.200Z",
|
||||
"context": {
|
||||
"message_id": "msg-ajecodZjyK",
|
||||
"trace_id": "CQZp9tGQTTowij",
|
||||
"reply_type": "clarification_required",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "Покажи контрагентов, по которым сальдо скорее всего не совпадет с их актом сверки. Может, стоит поторопиться и запросить сверку?",
|
||||
"answer_text": "Коротко: Наиболее вероятный механизм проблемы подтвержден частично и требует первичной проверки. Что сломано:\n- Есть признаки проблемы, но без уточнений по периоду и объекту вывод ненадежен. Почему это похоже на проблему:\n- Сигнал проблемы есть, но механизм подтвержден не полностью. На чем это основано:\n- Опора частичная: часть требований покрыта не полностью.\n- Отдельно не подтверждено или покрыто частично: R1. Что проверить первым:\n- Уточните период проверки, чтобы подтвердить проблему без лишнего шума.\n- Проверьте связку документов и проводок по проблемному участку в указанном периоде.\n- Уточните период проверки (например, июль 2020).\n- Уточните счет или группу счетов (например, 19, 60, 62). Ограничения:\n- Ни одно требование не получило подтвержденного покрытия.\n- Нет explainable-сигналов why_included в результатах выборки.\n- Недостаточно покрытия required anchors для claim-bound grounding.\n- Недостаточно допустимого evidence для обоснованного ответа.\n- Targeted evidence acquisition не дал допустимых попаданий по claim target path."
|
||||
}
|
||||
},
|
||||
{
|
||||
"annotation_id": "ann-mnrsvcxx-o0igav4",
|
||||
"run_id": "assistant-stage1-rDM8XCbg7q",
|
||||
"case_id": "AUTO-004",
|
||||
"session_id": "assistant-stage1-rDM8XCbg7q-AUTO-004",
|
||||
"message_index": 1,
|
||||
"rating": 1,
|
||||
"comment": "Тут опять очевидно очень прямой и конкретный запрос, и мы его должны вроде, по идее, как отработать. Он у нас уже вроде должен отрабатываться. Если нет, то значит надо его... Так, надо расширить маршрутизацию. То есть глобально надо найти, короче, документы на какой-то период определенный, если мы говорим про срез. Надо найти документы, которые не закрыты. Вот и всё. Как бы это очевидно. Ну, как бы не закрытые доки просто ищем и выводим. Вот. Если таких нету, то, значит, отвечаем, что такая проблема отсутствует.",
|
||||
"manual_case_decision": "needs_routing_extension",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T18:18:02.949Z",
|
||||
"updated_at": "2026-04-09T18:18:02.949Z",
|
||||
"context": {
|
||||
"message_id": "msg-fIfli9GAKl",
|
||||
"trace_id": "address-0wCj3SAhfK",
|
||||
"reply_type": "partial_coverage",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "Где у нас есть платежи, но нет документов для закрытия взаиморасчетов? Это уже требует ручной проверки.",
|
||||
"answer_text": "Для точного адресного поиска не хватает обязательного якоря.\nПричина: контрагент по указанному имени/алиасу не найден в materialized live-строках.\nЧто нужно уточнить: уточните точное имя контрагента или добавьте ИНН."
|
||||
}
|
||||
},
|
||||
{
|
||||
"annotation_id": "ann-mnrsx62p-sp9mrfy",
|
||||
"run_id": "assistant-stage1-rDM8XCbg7q",
|
||||
"case_id": "AUTO-005",
|
||||
"session_id": "assistant-stage1-rDM8XCbg7q-AUTO-005",
|
||||
"message_index": 1,
|
||||
"rating": 1,
|
||||
"comment": "Опять точно такая же проблема. Ну как бы очень технический ответ про якоря и про обязательства. Это уже обсуждали. И никакие уточнения, а почему модель просто какие-то уточнения, это противоречит вообще-то изначальному запросу. Запрос максимально простой. Надо найти документы, которые не оплачены. То есть договора мертвые, без оплат. Вот и всё. У нас точно такие есть, и их точно довольно-таки легко маршрутизировать, если они у нас не маршрутизируются.",
|
||||
"manual_case_decision": "needs_routing_extension",
|
||||
"annotation_author": "manual_reviewer",
|
||||
"resolved": false,
|
||||
"resolved_at": null,
|
||||
"resolved_by": null,
|
||||
"created_at": "2026-04-09T18:19:27.359Z",
|
||||
"updated_at": "2026-04-09T20:06:33.494Z",
|
||||
"context": {
|
||||
"message_id": "msg-uVQqIiArS7",
|
||||
"trace_id": "address-2NS8zxTvo5",
|
||||
"reply_type": "partial_coverage",
|
||||
"eval_target": "assistant_stage1",
|
||||
"prompt_version": "address_query_runtime_v1",
|
||||
"domain": null,
|
||||
"query_class": null,
|
||||
"question_text": "По каким контрагентам документы есть, а оплат нет. Может, стоит взять на карандаш такие ситуации чтоб не тянуть дальше?",
|
||||
"answer_text": "Для точного адресного поиска не хватает обязательного якоря.\nПричина: контрагент по указанному имени/алиасу не найден в materialized live-строках.\nЧто нужно уточнить: уточните точное имя контрагента или добавьте ИНН."
|
||||
}
|
||||
}
|
||||
]
|
||||
File diff suppressed because one or more lines are too long
+878
File diff suppressed because one or more lines are too long
+46
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrmoiey-j9akyvu",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T15:24:45.754Z",
|
||||
"generation_id": "gen-mnrmoiey-j9akyvu",
|
||||
"mode": "qwen_seed",
|
||||
"domain": "settlements",
|
||||
"scenario_count": 2,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_settlements",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Кто из поставщиков имеет хвосты с документами на конец месяца, которые уже больше похожи на систематическую проблему, а не на обычную задержку?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_settlements",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть реализации, которые могут портить отчетность по выручке, если их не проверить до конца периода?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrnrwtc-za8a8o0",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T15:55:24.001Z",
|
||||
"generation_id": "gen-mnrnrwtc-za8a8o0",
|
||||
"mode": "qwen_seed",
|
||||
"domain": null,
|
||||
"scenario_count": 20,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005",
|
||||
"AUTO-006",
|
||||
"AUTO-007",
|
||||
"AUTO-008",
|
||||
"AUTO-009",
|
||||
"AUTO-010",
|
||||
"AUTO-011",
|
||||
"AUTO-012",
|
||||
"AUTO-013",
|
||||
"AUTO-014",
|
||||
"AUTO-015",
|
||||
"AUTO-016",
|
||||
"AUTO-017",
|
||||
"AUTO-018",
|
||||
"AUTO-019",
|
||||
"AUTO-020"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие поставщики уже пару месяцев держат хвосты, которые выглядят как системная проблема, а не просто задержка с документами?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас покупатели 'отгрузили - денег нет - закрытия нет' и нужна ручная проверка этих контрагентов?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов, чьё сальдо скорее всего не совпадет с их актом сверки, если его запросить прямо сейчас."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть оплаты, но документов для закрытия взаиморасчетов нет совсем?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие контрагенты имеют документы, но нормального закрытия оплатами не видно?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-006",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли зависшие авансы, которые давно требуют перепроверки или окончательного закрытия?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-007",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие реализации на конец периода выглядят так, будто они зависли и портят картину по выручке?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-008",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "По каким отгрузкам видно, что проблема не просто в том, что клиент не оплатил, а в том, что связка документов собрана криво?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-009",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие накладные на складе уже давно не сопровождаются поступлениями или отправками - это может быть подозрительно?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-010",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли контрагенты, у которых есть отгрузки без связанных оплат и их нужно проверить на наличие долгов?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-011",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие документы покупок были созданы давно, но не закрыты вводом накладных или актов - это может быть прямой риск?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-012",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас зависли авансовые отгрузки с датами старше полугода и их нужно либо списать, либо проверить детали?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-013",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие поставщики уже давно не подтверждали свои счета - это может указывать на проблемы в цепочке взаиморасчетов?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-014",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли контрагенты с отгрузками или покупками, где документы есть, а реальных транзакций нет - это может быть фиктивным?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-015",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие реализации уже подтверждены, но их сальдо не соотносится с фактической выручкой за период?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-016",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас зависли авансы поставщикам и они требуют ручной проверки на предмет реальных платежей?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-017",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли контрагенты, у которых есть отгрузки без связанных документов или платежей - это может быть проблемой для баланса?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-018",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие документы покупок не завершены вводом актов и их сальдо выглядит подозрительно?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-019",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас зависли авансовые поступления от клиентов, которые давно требуют проверки или списания?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-020",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли контрагенты, у которых есть реализации без связанных оплат - это может быть фиктивным?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrrdfbj-mrcxcjg",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T17:36:06.607Z",
|
||||
"generation_id": "gen-mnrrdfbj-mrcxcjg",
|
||||
"mode": "qwen_seed",
|
||||
"domain": null,
|
||||
"scenario_count": 10,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005",
|
||||
"AUTO-006",
|
||||
"AUTO-007",
|
||||
"AUTO-008",
|
||||
"AUTO-009",
|
||||
"AUTO-010"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие покупатели на конец месяца держат нас в незакрытых взаиморасчетах уже больше чем обычно, и это начинает напоминать реальные проблемы а не просто задержку?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас с поставщиками видны хвосты по документам, которые явно вышли за рамки обычной задержки?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие реализации на конец периода еще не закрыты и могут испортить картину при ревизии?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов у которых сальдо скорее всего не совпадет с их актом сверки, если его запросить прямо сейчас."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где по покупателям есть история отгрузили - денег нет - закрытия нет и это уже требует ручной проверки?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-006",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли зависшие авансы которые уже давно надо было либо закрыть, либо хотя бы перепроверить руками?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-007",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие реализации на конец периода выглядят так будто они зависли и будут портить картину по выручке если их не проверить заранее?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-008",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть оплаты но не хватает документов чтобы закрыть взаиморасчеты?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-009",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие контрагенты показывают документы, но нормального закрытия оплатами нет - это требует ручной проверки?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-010",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Кто из контрагентов держит нас в незакрытых взаиморасчетах на конец месяца и это начинает напоминать реальные проблемы а не просто задержку?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrshzcm-xyiv4gs",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T18:07:38.806Z",
|
||||
"generation_id": "gen-mnrshzcm-xyiv4gs",
|
||||
"mode": "qwen_seed",
|
||||
"domain": null,
|
||||
"scenario_count": 10,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005",
|
||||
"AUTO-006",
|
||||
"AUTO-007",
|
||||
"AUTO-008",
|
||||
"AUTO-009",
|
||||
"AUTO-010"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие поставщики у нас уже пару месяцев сдают акты без приходок. Может, их надо проконтролировать отдельно чтоб не засорять бухгалтерию дальше?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас висят покупатели со слишком длинным периодом между отправкой товара и его оплатой, и это уже вызывает тревогу?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов, по которым сальдо скорее всего не совпадет с их актом сверки. Может, стоит поторопиться и запросить сверку?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть платежи, но нет документов для закрытия взаиморасчетов? Это уже требует ручной проверки."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "По каким контрагентам документы есть, а оплат нет. Может, стоит взять на карандаш такие ситуации чтоб не тянуть дальше?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-006",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли зависшие авансы, которые нужно либо закрыть, либо перепроверить уже давно?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-007",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие реализации на конец периода выглядят так, будто они зависли и могут портить картину по выручке?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-008",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас отгрузки с кривыми документами. Это уже требует ручного анализа."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-009",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Кто из контрагентов давно не подтверждал свои расчеты. Может, стоит напомнить о сверках?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-010",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие покупатели долго задерживают оплату без явных причин. Это тоже требует внимания."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrvqxcg-wa3jsro",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T19:38:34.960Z",
|
||||
"generation_id": "gen-mnrvqxcg-wa3jsro",
|
||||
"mode": "qwen_seed",
|
||||
"domain": null,
|
||||
"scenario_count": 5,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие у нас поставщики пока вообще никак не проявились в текущем месяце и это уже начинает выглядеть подозрительно?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть реализации, которые сидят без закрытий на конец отчетного периода, и они реально могут испортить финансовую картину?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов, по которым сальдо в 1С явно расходится с тем, что должно быть по данным их последнего акта сверки."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли такие авансы, которые уже давно не используются и их пора или списать, или перепроверить?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас есть оплаты за товары/услуги, но самих документов на эти платежи до сих пор нет в системе?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_gen-mnrvs132-1dewq5r",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_suite_v0_1",
|
||||
"generated_at": "2026-04-09T19:39:26.463Z",
|
||||
"generation_id": "gen-mnrvs132-1dewq5r",
|
||||
"mode": "qwen_seed",
|
||||
"domain": null,
|
||||
"scenario_count": 5,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие покупатели оставили хвосты по отгрузкам на конец месяца, которые скорее говорят про проблемы с документами, чем просто задержку платежей?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас висят неоплаченные реализации, что могут испортить баланс выручки, если их не проверять заранее?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов, по которым сальдо явно расходится с тем, что они напишут в сверке, если её запросить сейчас."
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли авансы, которые уже давно не закрыты и требуют ручной перепроверки?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "qwen_seed_general",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие поставщики оставили хвосты по документам на конец месяца, что указывают скорее на проблемы с оформлением, чем на задержку?"
|
||||
}
|
||||
],
|
||||
"expected_hints": {
|
||||
"expected_reply_type": null,
|
||||
"expected_degraded_to": null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"suite_id": "assistant_autogen_runtime_job-O2V_MDy1qP",
|
||||
"suite_version": "0.1.0",
|
||||
"schema_version": "assistant_autogen_runtime_v0_1",
|
||||
"scenario_count": 5,
|
||||
"case_ids": [
|
||||
"AUTO-001",
|
||||
"AUTO-002",
|
||||
"AUTO-003",
|
||||
"AUTO-004",
|
||||
"AUTO-005"
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "AUTO-001",
|
||||
"scenario_tag": "autogen_runtime",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие покупатели оставили хвосты по отгрузкам на конец месяца, которые скорее говорят про проблемы с документами, чем просто задержку платежей?"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-002",
|
||||
"scenario_tag": "autogen_runtime",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Где у нас висят неоплаченные реализации, что могут испортить баланс выручки, если их не проверять заранее?"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-003",
|
||||
"scenario_tag": "autogen_runtime",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Покажи контрагентов, по которым сальдо явно расходится с тем, что они напишут в сверке, если её запросить сейчас."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-004",
|
||||
"scenario_tag": "autogen_runtime",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Есть ли авансы, которые уже давно не закрыты и требуют ручной перепроверки?"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"case_id": "AUTO-005",
|
||||
"scenario_tag": "autogen_runtime",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"turns": [
|
||||
{
|
||||
"user_message": "Какие поставщики оставили хвосты по документам на конец месяца, что указывают скорее на проблемы с оформлением, чем на задержку?"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"run_id": "eval-WJUoAGRQua",
|
||||
"timestamp": "2026-04-09T18:01:58.497Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"hybrid_store_plus_live": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "JDGrTt0tveFgfh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "qsdm7lX8NXx1-C",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
{
|
||||
"run_id": "eval-YIS0RgjMsg",
|
||||
"timestamp": "2026-04-09T18:01:58.499Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"hybrid_store_plus_live": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "nl9np-2vh2tA1J",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "e6AdKc4cEUNvoK",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-snt_txRT-J",
|
||||
"timestamp": "2026-04-09T18:01:39.193Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "LcMJHaAgya9hgV",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "DB2D2zvB92-2b9",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "2kWFwub_GELA7S",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
+2
-2
@@ -4,8 +4,8 @@
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>NDC AI Normalizer Playground</title>
|
||||
<script type="module" crossorigin src="/assets/index-Cbn_mHUl.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-BT0bMOoF.css">
|
||||
<script type="module" crossorigin src="/assets/index-DNcr9aV9.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-qw2gR5-7.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import type {
|
||||
AsyncEvalRunStartResponse,
|
||||
AsyncEvalRunStatusResponse,
|
||||
AutoGenPersonalityCatalogResponse,
|
||||
AutoGenHistoryResponse,
|
||||
AutoGenMode,
|
||||
AutoRunAnnotationsResponse,
|
||||
@@ -155,6 +158,8 @@ export const apiClient = {
|
||||
mode?: "standard" | "single-pass-strict";
|
||||
caseSetFile?: string;
|
||||
rawQuestions?: string;
|
||||
evalTarget?: "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0";
|
||||
compareWithReportFile?: string;
|
||||
}): Promise<{ ok: boolean; report: unknown }> {
|
||||
return request("/eval/run", {
|
||||
method: "POST",
|
||||
@@ -176,11 +181,58 @@ export const apiClient = {
|
||||
useMock: Boolean(input.useMock),
|
||||
mode: input.mode ?? "standard",
|
||||
caseSetFile: input.caseSetFile,
|
||||
rawQuestions: input.rawQuestions
|
||||
rawQuestions: input.rawQuestions,
|
||||
eval_target: input.evalTarget,
|
||||
compare_with_report_file: input.compareWithReportFile
|
||||
})
|
||||
});
|
||||
},
|
||||
|
||||
async startEvalRunAsync(input: {
|
||||
connection: ConnectionState;
|
||||
prompts: PromptState;
|
||||
promptVersion?: string;
|
||||
caseIds?: string[];
|
||||
useMock?: boolean;
|
||||
mode?: "standard" | "single-pass-strict";
|
||||
caseSetFile?: string;
|
||||
rawQuestions?: string;
|
||||
evalTarget?: "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0";
|
||||
compareWithReportFile?: string;
|
||||
questions?: string[];
|
||||
}): Promise<AsyncEvalRunStartResponse> {
|
||||
return request("/eval/run-async/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
normalizeConfig: {
|
||||
llmProvider: input.connection.llmProvider,
|
||||
apiKey: input.connection.apiKey,
|
||||
model: input.connection.model,
|
||||
baseUrl: input.connection.baseUrl,
|
||||
temperature: input.connection.temperature,
|
||||
maxOutputTokens: input.connection.maxOutputTokens,
|
||||
promptVersion: input.promptVersion,
|
||||
systemPrompt: input.prompts.systemPrompt,
|
||||
developerPrompt: input.prompts.developerPrompt,
|
||||
domainPrompt: input.prompts.domainPrompt,
|
||||
fewShotExamples: input.prompts.fewShotExamples
|
||||
},
|
||||
caseIds: input.caseIds,
|
||||
useMock: Boolean(input.useMock),
|
||||
mode: input.mode ?? "standard",
|
||||
caseSetFile: input.caseSetFile,
|
||||
rawQuestions: input.rawQuestions,
|
||||
eval_target: input.evalTarget,
|
||||
compare_with_report_file: input.compareWithReportFile,
|
||||
questions: input.questions
|
||||
})
|
||||
});
|
||||
},
|
||||
|
||||
async loadEvalRunAsyncStatus(jobId: string): Promise<AsyncEvalRunStatusResponse> {
|
||||
return request(`/eval/run-async/${encodeURIComponent(jobId)}`);
|
||||
},
|
||||
|
||||
async startRun(): Promise<{ ok: boolean; run: RuntimeRun; runId: string; sessionId: string; status: string }> {
|
||||
return request("/accounting-agent/v1/runs/start", {
|
||||
method: "POST",
|
||||
@@ -321,6 +373,20 @@ export const apiClient = {
|
||||
});
|
||||
},
|
||||
|
||||
async updateAutoRunAnnotation(input: {
|
||||
annotation_id: string;
|
||||
resolved: boolean;
|
||||
resolved_by?: string;
|
||||
}): Promise<{ ok: boolean; annotation: AutoRunAnnotationRecord; case_annotation_stats: { count: number; latest_at: string | null; avg_rating: number | null } | null }> {
|
||||
return request(`/autoruns/annotations/${encodeURIComponent(input.annotation_id)}`, {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify({
|
||||
resolved: input.resolved,
|
||||
resolved_by: input.resolved_by
|
||||
})
|
||||
});
|
||||
},
|
||||
|
||||
async loadAutoRunPostAnalysis(input?: {
|
||||
run_id?: string;
|
||||
limit_per_queue?: number;
|
||||
@@ -359,12 +425,24 @@ export const apiClient = {
|
||||
return request(`/autoruns/autogen/history${query ? `?${query}` : ""}`);
|
||||
},
|
||||
|
||||
async loadAutoRunAutogenPersonalityCatalog(): Promise<AutoGenPersonalityCatalogResponse> {
|
||||
return request("/autoruns/autogen/personality-catalog");
|
||||
},
|
||||
|
||||
async generateAutoRunQuestions(input: {
|
||||
mode: AutoGenMode;
|
||||
count: number;
|
||||
domain?: string;
|
||||
persist_to_eval_cases?: boolean;
|
||||
generated_by?: string;
|
||||
llm?: {
|
||||
llm_provider?: "openai" | "local";
|
||||
api_key?: string;
|
||||
model?: string;
|
||||
base_url?: string;
|
||||
temperature?: number;
|
||||
max_output_tokens?: number;
|
||||
};
|
||||
context?: {
|
||||
llm_provider?: string;
|
||||
model?: string;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,4 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { PanelFrame } from "./PanelFrame";
|
||||
import type { ConnectionState } from "../state/types";
|
||||
|
||||
@@ -26,6 +27,46 @@ export function ConnectionPanel({
|
||||
}: ConnectionPanelProps) {
|
||||
const isLocal = value.llmProvider === "local";
|
||||
const modelInCatalog = modelOptions.includes(value.model);
|
||||
const [temperatureInput, setTemperatureInput] = useState(String(value.temperature));
|
||||
const [maxOutputTokensInput, setMaxOutputTokensInput] = useState(String(value.maxOutputTokens));
|
||||
|
||||
useEffect(() => {
|
||||
setTemperatureInput(String(value.temperature));
|
||||
}, [value.temperature]);
|
||||
|
||||
useEffect(() => {
|
||||
setMaxOutputTokensInput(String(value.maxOutputTokens));
|
||||
}, [value.maxOutputTokens]);
|
||||
|
||||
const commitTemperatureInput = (raw: string) => {
|
||||
const normalized = raw.replace(",", ".").trim();
|
||||
if (!normalized) {
|
||||
setTemperatureInput(String(value.temperature));
|
||||
return;
|
||||
}
|
||||
const parsed = Number(normalized);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
setTemperatureInput(String(value.temperature));
|
||||
return;
|
||||
}
|
||||
onChange({ ...value, temperature: parsed });
|
||||
setTemperatureInput(String(parsed));
|
||||
};
|
||||
|
||||
const commitMaxOutputTokensInput = (raw: string) => {
|
||||
const normalized = raw.trim();
|
||||
if (!normalized) {
|
||||
setMaxOutputTokensInput(String(value.maxOutputTokens));
|
||||
return;
|
||||
}
|
||||
const parsed = Number.parseInt(normalized, 10);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
setMaxOutputTokensInput(String(value.maxOutputTokens));
|
||||
return;
|
||||
}
|
||||
onChange({ ...value, maxOutputTokens: parsed });
|
||||
setMaxOutputTokensInput(String(parsed));
|
||||
};
|
||||
|
||||
return (
|
||||
<PanelFrame
|
||||
@@ -108,8 +149,14 @@ export function ConnectionPanel({
|
||||
<input
|
||||
type="number"
|
||||
step="0.1"
|
||||
value={value.temperature}
|
||||
onChange={(event) => onChange({ ...value, temperature: Number(event.target.value) })}
|
||||
value={temperatureInput}
|
||||
onChange={(event) => setTemperatureInput(event.target.value)}
|
||||
onBlur={(event) => commitTemperatureInput(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
commitTemperatureInput((event.target as HTMLInputElement).value);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
|
||||
@@ -117,8 +164,14 @@ export function ConnectionPanel({
|
||||
Max output tokens
|
||||
<input
|
||||
type="number"
|
||||
value={value.maxOutputTokens}
|
||||
onChange={(event) => onChange({ ...value, maxOutputTokens: Number(event.target.value) })}
|
||||
value={maxOutputTokensInput}
|
||||
onChange={(event) => setMaxOutputTokensInput(event.target.value)}
|
||||
onBlur={(event) => commitMaxOutputTokensInput(event.target.value)}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
commitMaxOutputTokensInput((event.target as HTMLInputElement).value);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
@@ -188,6 +188,9 @@ export interface AutoRunAnnotationRecord {
|
||||
comment: string;
|
||||
manual_case_decision: ManualCaseDecision;
|
||||
annotation_author: string | null;
|
||||
resolved: boolean;
|
||||
resolved_at: string | null;
|
||||
resolved_by: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
context: {
|
||||
@@ -198,6 +201,8 @@ export interface AutoRunAnnotationRecord {
|
||||
prompt_version: string | null;
|
||||
domain: string | null;
|
||||
query_class: string | null;
|
||||
question_text: string | null;
|
||||
answer_text: string | null;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -243,6 +248,8 @@ export interface AutoRunDialogMessage {
|
||||
trace_id: string | null;
|
||||
reply_type: string | null;
|
||||
message_index: number;
|
||||
case_id?: string | null;
|
||||
case_message_index?: number | null;
|
||||
commented: boolean;
|
||||
annotation: AutoRunAnnotationRecord | null;
|
||||
}
|
||||
@@ -251,7 +258,7 @@ export interface AutoRunDialogResponse {
|
||||
ok: boolean;
|
||||
run_id: string;
|
||||
case_id: string;
|
||||
source: "assistant_session" | "report_fallback" | "none";
|
||||
source: "assistant_session" | "report_fallback" | "run_aggregate" | "none";
|
||||
session_id: string;
|
||||
messages: AutoRunDialogMessage[];
|
||||
decomposition: string[];
|
||||
@@ -320,6 +327,8 @@ export interface AutoGenHistoryRecord {
|
||||
assistant_prompt_version: string | null;
|
||||
decomposition_prompt_version: string | null;
|
||||
prompt_fingerprint: string | null;
|
||||
autogen_personality_id: string | null;
|
||||
autogen_personality_prompt: string | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
@@ -329,6 +338,57 @@ export interface AutoGenHistoryResponse {
|
||||
items: AutoGenHistoryRecord[];
|
||||
}
|
||||
|
||||
export interface AutoGenPersonalityDefinition {
|
||||
id: string;
|
||||
label: string;
|
||||
domain: string | null;
|
||||
default_prompt: string;
|
||||
source: "built_in" | "capabilities_registry";
|
||||
}
|
||||
|
||||
export interface AutoGenPersonalityCatalogResponse {
|
||||
ok: boolean;
|
||||
generated_at: string;
|
||||
items: AutoGenPersonalityDefinition[];
|
||||
}
|
||||
|
||||
export interface AsyncEvalRunCase {
|
||||
case_id: string;
|
||||
turns_total: number;
|
||||
status: "queued" | "running" | "completed" | "failed";
|
||||
messages: AutoRunDialogMessage[];
|
||||
}
|
||||
|
||||
export interface AsyncEvalRunJob {
|
||||
job_id: string;
|
||||
status: "queued" | "running" | "completed" | "failed";
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
eval_target: AutoRunTarget;
|
||||
run_id: string;
|
||||
case_set_file: string | null;
|
||||
total_cases: number;
|
||||
completed_cases: number;
|
||||
error: string | null;
|
||||
cases: AsyncEvalRunCase[];
|
||||
report_summary: {
|
||||
run_id: string | null;
|
||||
run_timestamp: string | null;
|
||||
score_index: number | null;
|
||||
cases_total: number | null;
|
||||
} | null;
|
||||
}
|
||||
|
||||
export interface AsyncEvalRunStartResponse {
|
||||
ok: boolean;
|
||||
job: AsyncEvalRunJob;
|
||||
}
|
||||
|
||||
export interface AsyncEvalRunStatusResponse {
|
||||
ok: boolean;
|
||||
job: AsyncEvalRunJob;
|
||||
}
|
||||
|
||||
export type AssistantFallbackType = "none" | "out_of_scope" | "clarification" | "partial" | "unknown";
|
||||
export type AssistantReplyType =
|
||||
| "factual"
|
||||
|
||||
@@ -741,10 +741,17 @@ button:disabled {
|
||||
padding: 10px;
|
||||
display: grid;
|
||||
gap: 5px;
|
||||
transition: background-color 0.2s ease, box-shadow 0.2s ease;
|
||||
}
|
||||
|
||||
.autoruns-run-item.selected {
|
||||
border-color: var(--line-strong);
|
||||
background: rgb(var(--rgb-active));
|
||||
color: rgb(var(--rgb-active-text));
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.autoruns-run-item.selected .autoruns-run-meta {
|
||||
color: rgba(var(--rgb-active-text), 0.95);
|
||||
}
|
||||
|
||||
.autoruns-run-head,
|
||||
@@ -790,7 +797,9 @@ button:disabled {
|
||||
}
|
||||
|
||||
.autoruns-case-item.selected {
|
||||
border-color: var(--line-strong);
|
||||
background: rgb(var(--rgb-active));
|
||||
color: rgb(var(--rgb-active-text));
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.autoruns-dialog-view {
|
||||
@@ -831,6 +840,17 @@ button:disabled {
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.autoruns-msg-case-tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
border-radius: 999px;
|
||||
padding: 2px 8px;
|
||||
font-size: 0.7rem;
|
||||
line-height: 1;
|
||||
color: rgb(var(--rgb-active-text));
|
||||
background: rgba(var(--rgb-active), 0.24);
|
||||
}
|
||||
|
||||
.autoruns-msg p {
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
@@ -840,30 +860,106 @@ button:disabled {
|
||||
|
||||
.autoruns-comment-icon {
|
||||
border: none;
|
||||
background: rgb(var(--rgb-surface-horizontal));
|
||||
color: var(--text-main);
|
||||
border-radius: 999px;
|
||||
min-width: 28px;
|
||||
min-height: 28px;
|
||||
padding: 0 8px;
|
||||
background: transparent;
|
||||
color: rgb(var(--rgb-text-main));
|
||||
border-radius: 0;
|
||||
min-width: 20px;
|
||||
min-height: 20px;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
padding: 0;
|
||||
line-height: 1;
|
||||
box-shadow: none;
|
||||
transform: none;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.autoruns-comment-icon:hover {
|
||||
border-color: var(--line-strong);
|
||||
background: transparent;
|
||||
color: rgb(var(--rgb-active));
|
||||
box-shadow: none;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.autoruns-comment-icon.commented {
|
||||
border-color: var(--line-strong);
|
||||
color: var(--lime-main);
|
||||
color: rgb(var(--rgb-active-text));
|
||||
background: transparent;
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
.comment-icon-svg {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
stroke: currentColor;
|
||||
stroke-width: 1.75;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
fill: none;
|
||||
}
|
||||
|
||||
.comment-icon-svg .comment-icon-dot {
|
||||
fill: currentColor;
|
||||
}
|
||||
|
||||
.comment-icon-svg.commented {
|
||||
fill: rgb(var(--rgb-active));
|
||||
stroke: rgb(var(--rgb-active));
|
||||
}
|
||||
|
||||
.autoruns-comment-icon.commented .comment-icon-dot {
|
||||
fill: rgb(var(--rgb-active-text));
|
||||
}
|
||||
|
||||
.autoruns-resolve-toggle {
|
||||
border: none;
|
||||
background: rgb(var(--rgb-surface-focus));
|
||||
color: rgb(var(--rgb-text-main));
|
||||
border-radius: 999px;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
min-width: 22px;
|
||||
min-height: 22px;
|
||||
padding: 0;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
box-shadow: none;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.autoruns-resolve-toggle:hover {
|
||||
background: rgb(var(--rgb-surface-focus));
|
||||
color: rgb(var(--rgb-active));
|
||||
box-shadow: none;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.autoruns-resolve-toggle:disabled {
|
||||
opacity: 0.55;
|
||||
cursor: wait;
|
||||
}
|
||||
|
||||
.autoruns-resolve-toggle.resolved {
|
||||
background: rgb(var(--rgb-active));
|
||||
color: rgb(var(--rgb-active-text));
|
||||
}
|
||||
|
||||
.resolve-icon-svg {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
fill: none;
|
||||
stroke: currentColor;
|
||||
stroke-width: 1.8;
|
||||
stroke-linecap: round;
|
||||
stroke-linejoin: round;
|
||||
}
|
||||
|
||||
.resolve-icon-svg.resolved {
|
||||
fill: currentColor;
|
||||
}
|
||||
|
||||
.autoruns-msg-annotation {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
@@ -882,6 +978,7 @@ button:disabled {
|
||||
flex: 1 1 auto;
|
||||
overflow: auto;
|
||||
padding-right: 2px;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.autoruns-autogen-list {
|
||||
@@ -900,6 +997,17 @@ button:disabled {
|
||||
padding: 8px;
|
||||
display: grid;
|
||||
gap: 5px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.autoruns-autogen-item.selected {
|
||||
background: rgb(var(--rgb-active));
|
||||
color: rgb(var(--rgb-active-text));
|
||||
}
|
||||
|
||||
.autoruns-autogen-item.selected .autoruns-run-meta,
|
||||
.autoruns-autogen-item.selected p {
|
||||
color: rgba(var(--rgb-active-text), 0.95);
|
||||
}
|
||||
|
||||
.autoruns-autogen-item header {
|
||||
@@ -916,6 +1024,58 @@ button:disabled {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
.autoruns-generated-questions {
|
||||
border: none;
|
||||
border-radius: 10px;
|
||||
background: rgb(var(--rgb-surface-horizontal));
|
||||
padding: 8px;
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.autoruns-generated-questions-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.autoruns-generated-questions-list {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
max-height: 220px;
|
||||
overflow: auto;
|
||||
padding-right: 2px;
|
||||
}
|
||||
|
||||
.autoruns-generated-question-item {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
border: none;
|
||||
border-radius: 9px;
|
||||
background: rgb(var(--rgb-surface-focus));
|
||||
padding: 6px 8px;
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.autoruns-generated-question-item span {
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.autoruns-remove-question-btn {
|
||||
flex: 0 0 auto;
|
||||
border: none;
|
||||
border-radius: 7px;
|
||||
background: rgb(var(--rgb-surface-horizontal));
|
||||
color: var(--text-main);
|
||||
min-width: 24px;
|
||||
height: 24px;
|
||||
font-size: 0.75rem;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.autoruns-comment-item {
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
@@ -926,6 +1086,7 @@ button:disabled {
|
||||
padding: 9px;
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.autoruns-comment-item p {
|
||||
@@ -936,16 +1097,48 @@ button:disabled {
|
||||
}
|
||||
|
||||
.autoruns-comment-item.selected {
|
||||
border-color: var(--line-strong);
|
||||
background: rgb(var(--rgb-active));
|
||||
color: rgb(var(--rgb-active-text));
|
||||
}
|
||||
|
||||
.autoruns-comment-item.selected p,
|
||||
.autoruns-comment-item.selected .autoruns-run-meta,
|
||||
.autoruns-comment-item.selected .muted {
|
||||
color: rgba(var(--rgb-active-text), 0.94);
|
||||
}
|
||||
|
||||
.autoruns-comment-item.selected .autoruns-resolve-toggle {
|
||||
background: rgba(var(--rgb-active-text), 0.18);
|
||||
color: rgb(var(--rgb-active-text));
|
||||
}
|
||||
|
||||
.autoruns-comment-head {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.autoruns-comment-head-actions {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.autoruns-comment-filter-row {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
align-items: end;
|
||||
gap: 10px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.autoruns-resolved-filter-toggle {
|
||||
min-height: 38px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.autoruns-msg.assistant {
|
||||
margin-right: 12%;
|
||||
}
|
||||
@@ -1096,7 +1289,8 @@ button:disabled {
|
||||
|
||||
.autoruns-form-grid,
|
||||
.autoruns-dialog-toolbar,
|
||||
.autoruns-stats-grid {
|
||||
.autoruns-stats-grid,
|
||||
.autoruns-comment-filter-row {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user