SILVER/NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ЮИ - Добавить удаление сохраненных наборов автопрогонов с удалением файлов на бэке

Browse Source
This commit is contained in:
dctouch
2026-04-16 21:02:30 +03:00
parent f3255cb3b8
commit a3a61b3a0f
22 changed files with 1895 additions and 1996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm_normalizer/backend/dist/routes/autoRuns.js
Vendored
+290 -12
View File
@@ -93,6 +93,16 @@ function clampInt(value, min, max, fallback) {
return max;
return rounded;
}
function isAutoGenMode(value) {
return value === "qwen_seed" || value === "codex_creative" || value === "saved_user_sessions";
}
function parseAutoGenTitle(value) {
const title = toStringSafe(value);
if (!title) {
return null;
}
return title.slice(0, 160);
}
function parseManualCaseDecision(value, fallback = "needs_dialog_policy_fix") {
const normalized = toStringSafe(value);
if (!normalized)
@@ -151,15 +161,11 @@ function readAutoGenHistory() {
.map((item) => ({
generation_id: toStringSafe(item.generation_id) ?? "",
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
mode: toStringSafe(item.mode) ?? "codex_creative",
mode: isAutoGenMode(toStringSafe(item.mode)) ? toStringSafe(item.mode) : "codex_creative",
title: parseAutoGenTitle(item.title),
count: clampInt(toNumberSafe(item.count), 1, 300, 20),
domain: toStringSafe(item.domain),
questions: toArray(item.questions)
.map((q) => toStringSafe(q))
.filter((q) => q !== null)
.map((q) => sanitizeGeneratedQuestion(q))
.filter((q) => q.length > 0)
.slice(0, 500),
questions: parseAssistantSessionQuestions(item.questions),
generated_by: toStringSafe(item.generated_by),
saved_case_set_file: toStringSafe(item.saved_case_set_file),
context: toRecord(item.context)
@@ -174,7 +180,10 @@ function readAutoGenHistory() {
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
: null
: null,
source_session_id: toStringSafe(toRecord(item.context)?.source_session_id),
saved_session_file: toStringSafe(toRecord(item.context)?.saved_session_file),
saved_case_set_kind: toStringSafe(toRecord(item.context)?.saved_case_set_kind)
}
: null
}))
@@ -1057,7 +1066,7 @@ function parseDecisionFilter(value) {
}
function parseAutoGenMode(value) {
const normalized = toStringSafe(value)?.toLowerCase() ?? "";
if (normalized === "qwen_seed" || normalized === "codex_creative") {
if (normalized === "qwen_seed" || normalized === "codex_creative" || normalized === "saved_user_sessions") {
return normalized;
}
return "codex_creative";
@@ -1150,6 +1159,12 @@ function sanitizeGeneratedQuestion(value) {
.replace(/\s+/g, " ")
.trim();
}
function parseAssistantSessionQuestions(value) {
return toArray(value)
.map((item) => sanitizeGeneratedQuestion(typeof item === "string" ? item : ""))
.filter((item) => item.length > 0)
.slice(0, 500);
}
const AUTOGEN_QUESTION_PLACEHOLDER_PATTERN = /^(?:questions?|вопросы?|список\s+вопросов)$/iu;
const AUTOGEN_QUESTION_TAIL_PATTERNS = [
/^(?:без\s+воды|по\s+факту|и\s+коротко|коротко|прям(?:\s+)?сейчас|за\s+весь\s+период|по\s+делу)\??$/iu
@@ -1413,6 +1428,18 @@ function buildAutogenCaseSetFileName(mode, generationId) {
].join("");
return `assistant_autogen_${mode}_${stamp}_${generationId}.json`;
}
function buildSavedAssistantSessionSnapshotFileName(generationId) {
const now = new Date();
const stamp = [
now.getUTCFullYear(),
String(now.getUTCMonth() + 1).padStart(2, "0"),
String(now.getUTCDate()).padStart(2, "0"),
String(now.getUTCHours()).padStart(2, "0"),
String(now.getUTCMinutes()).padStart(2, "0"),
String(now.getUTCSeconds()).padStart(2, "0")
].join("");
return `assistant_saved_session_${stamp}_${generationId}.json`;
}
function buildAutogenCaseSetPayload(input) {
const normalizedQuestions = Array.from(new Set(input.questions.map((item) => sanitizeGeneratedQuestion(item)).filter((item) => item.length > 0)));
const cases = normalizedQuestions.map((question, index) => ({
@@ -1439,6 +1466,99 @@ function buildAutogenCaseSetPayload(input) {
cases
};
}
function buildSavedSessionCaseSetPayload(input) {
const questions = parseAssistantSessionQuestions(input.questions);
const turns = questions.map((question) => ({
user_message: question
}));
const caseId = "SAVED-001";
return {
suite_id: `assistant_saved_session_${input.generationId}`,
suite_version: "0.1.0",
schema_version: "assistant_saved_session_suite_v0_1",
generated_at: new Date().toISOString(),
generation_id: input.generationId,
mode: "saved_user_sessions",
title: input.title,
scenario_count: turns.length > 0 ? 1 : 0,
case_ids: turns.length > 0 ? [caseId] : [],
cases: turns.length > 0
? [
{
case_id: caseId,
scenario_tag: "saved_user_sessions",
title: input.title,
question_type: turns.length > 1 ? "followup" : "direct",
broadness_level: "medium",
turns
}
]
: []
};
}
function ensureDirSync(targetDir) {
if (!fs_1.default.existsSync(targetDir)) {
fs_1.default.mkdirSync(targetDir, { recursive: true });
}
}
function writeJsonFile(targetPath, payload) {
ensureDirSync(path_1.default.dirname(targetPath));
fs_1.default.writeFileSync(targetPath, JSON.stringify(payload, null, 2), "utf-8");
}
function rewriteAutoGenCaseSetFile(record) {
const caseSetFile = toStringSafe(record.saved_case_set_file);
if (!caseSetFile) {
return null;
}
const targetPath = path_1.default.resolve(config_1.EVAL_CASES_DIR, caseSetFile);
const payload = record.mode === "saved_user_sessions"
? buildSavedSessionCaseSetPayload({
generationId: record.generation_id,
title: record.title,
questions: record.questions
})
: buildAutogenCaseSetPayload({
generationId: record.generation_id,
mode: record.mode,
domain: record.domain,
questions: record.questions
});
writeJsonFile(targetPath, payload);
return caseSetFile;
}
function writeSavedAssistantSessionSnapshot(input) {
const fileName = buildSavedAssistantSessionSnapshotFileName(input.generationId);
const targetPath = path_1.default.resolve(path_1.default.dirname(config_1.AUTORUN_GENERATOR_HISTORY_FILE), "saved_sessions", fileName);
writeJsonFile(targetPath, {
saved_at: new Date().toISOString(),
generation_id: input.generationId,
mode: "saved_user_sessions",
title: input.title,
source_session_id: input.sessionId,
questions: input.questions,
session: input.session
});
return fileName;
}
function resolveFileInsideDir(baseDir, fileName) {
const normalized = toStringSafe(fileName);
if (!normalized) {
return null;
}
const targetPath = path_1.default.resolve(baseDir, normalized);
const relative = path_1.default.relative(baseDir, targetPath);
if (relative.startsWith("..") || path_1.default.isAbsolute(relative)) {
return null;
}
return targetPath;
}
function safeDeleteFile(targetPath) {
if (!targetPath || !fs_1.default.existsSync(targetPath)) {
return null;
}
fs_1.default.unlinkSync(targetPath);
return targetPath;
}
function collectPostAnalysis(annotations, runMap, limitPerQueue) {
const byDecision = {};
const byQueue = {};
@@ -1522,7 +1642,7 @@ function collectPostAnalysis(annotations, runMap, limitPerQueue) {
].slice(0, 60)
};
}
function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIResponsesClient()) {
function buildAutoRunsRouter(services, openaiClient = new openaiResponsesClient_1.OpenAIResponsesClient()) {
const router = (0, express_1.Router)();
router.get("/api/autoruns/history", (req, res) => {
const filters = parseFilters(req.query);
@@ -1884,7 +2004,7 @@ function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIRe
try {
const limit = clampInt(toNumberSafe(req.query.limit), 1, 500, 120);
const rawMode = toStringSafe(req.query.mode);
const includeAllModes = !rawMode || !["qwen_seed", "codex_creative"].includes(rawMode);
const includeAllModes = !rawMode || !isAutoGenMode(rawMode);
const modeFilter = rawMode ?? "codex_creative";
const items = readAutoGenHistory()
.filter((item) => (includeAllModes ? true : item.mode === modeFilter))
@@ -1911,6 +2031,157 @@ function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIRe
next(error);
}
});
router.post("/api/autoruns/autogen/save-assistant-session", (req, res, next) => {
try {
const body = toRecord(req.body);
if (!body) {
throw new http_1.ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "JSON body is required", 400);
}
const sessionId = toStringSafe(body.session_id);
const title = parseAutoGenTitle(body.title);
const generatedBy = parseAnnotationAuthor(body.generated_by);
const context = toRecord(body.context);
if (!sessionId) {
throw new http_1.ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "session_id is required", 400);
}
if (!title) {
throw new http_1.ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "title is required", 400);
}
const session = services.assistantService.getSession(sessionId);
if (!session) {
throw new http_1.ApiError("ASSISTANT_SESSION_NOT_FOUND", `Session not found: ${sessionId}`, 404);
}
const questions = session.items
.filter((item) => item.role === "user")
.map((item) => sanitizeGeneratedQuestion(item.text))
.filter((item) => item.length > 0);
if (questions.length === 0) {
throw new http_1.ApiError("ASSISTANT_SESSION_EMPTY", "Assistant session has no user questions to save.", 400);
}
const generationId = generateAutogenId();
const caseSetFile = buildAutogenCaseSetFileName("saved_user_sessions", generationId);
const caseSetPath = path_1.default.resolve(config_1.EVAL_CASES_DIR, caseSetFile);
writeJsonFile(caseSetPath, buildSavedSessionCaseSetPayload({
generationId,
title,
questions
}));
const snapshotFile = writeSavedAssistantSessionSnapshot({
generationId,
sessionId,
title,
session: session,
questions
});
const record = {
generation_id: generationId,
created_at: new Date().toISOString(),
mode: "saved_user_sessions",
title,
count: questions.length,
domain: null,
questions,
generated_by: generatedBy,
saved_case_set_file: caseSetFile,
context: {
llm_provider: toStringSafe(context?.llm_provider),
model: toStringSafe(context?.model),
assistant_prompt_version: toStringSafe(context?.assistant_prompt_version),
decomposition_prompt_version: toStringSafe(context?.decomposition_prompt_version),
prompt_fingerprint: toStringSafe(context?.prompt_fingerprint)
? repairAutogenMojibake(String(context?.prompt_fingerprint))
: null,
autogen_personality_id: null,
autogen_personality_prompt: null,
source_session_id: sessionId,
saved_session_file: snapshotFile,
saved_case_set_kind: "assistant_session_scenario"
}
};
const history = readAutoGenHistory();
history.unshift(record);
writeAutoGenHistory(history.slice(0, 500));
(0, http_1.ok)(res, {
ok: true,
generation: record
});
}
catch (error) {
next(error);
}
});
router.patch("/api/autoruns/autogen/history/:generation_id/questions", (req, res, next) => {
try {
const generationId = toStringSafe(req.params.generation_id);
const body = toRecord(req.body);
if (!generationId) {
throw new http_1.ApiError("INVALID_AUTOGEN_GENERATION_ID", "generation_id is required", 400);
}
if (!body) {
throw new http_1.ApiError("INVALID_AUTOGEN_QUESTIONS_PAYLOAD", "JSON body is required", 400);
}
const questions = parseAssistantSessionQuestions(body.questions);
if (questions.length === 0) {
throw new http_1.ApiError("INVALID_AUTOGEN_QUESTIONS_PAYLOAD", "questions must contain at least one item", 400);
}
const history = readAutoGenHistory();
const targetIndex = history.findIndex((item) => item.generation_id === generationId);
if (targetIndex < 0) {
throw new http_1.ApiError("AUTOGEN_GENERATION_NOT_FOUND", `Generation not found: ${generationId}`, 404);
}
const current = history[targetIndex];
const updated = {
...current,
count: questions.length,
questions
};
rewriteAutoGenCaseSetFile(updated);
history[targetIndex] = updated;
writeAutoGenHistory(history);
(0, http_1.ok)(res, {
ok: true,
generation: updated
});
}
catch (error) {
next(error);
}
});
router.delete("/api/autoruns/autogen/history/:generation_id", (req, res, next) => {
try {
const generationId = toStringSafe(req.params.generation_id);
if (!generationId) {
throw new http_1.ApiError("INVALID_AUTOGEN_GENERATION_ID", "generation_id is required", 400);
}
const history = readAutoGenHistory();
const targetIndex = history.findIndex((item) => item.generation_id === generationId);
if (targetIndex < 0) {
throw new http_1.ApiError("AUTOGEN_GENERATION_NOT_FOUND", `Generation not found: ${generationId}`, 404);
}
const target = history[targetIndex];
const deletedFiles = [];
const caseSetPath = resolveFileInsideDir(config_1.EVAL_CASES_DIR, target.saved_case_set_file);
const savedSessionPath = resolveFileInsideDir(path_1.default.resolve(path_1.default.dirname(config_1.AUTORUN_GENERATOR_HISTORY_FILE), "saved_sessions"), target.context?.saved_session_file ?? null);
const deletedCaseSet = safeDeleteFile(caseSetPath);
if (deletedCaseSet) {
deletedFiles.push(deletedCaseSet);
}
const deletedSavedSession = safeDeleteFile(savedSessionPath);
if (deletedSavedSession) {
deletedFiles.push(deletedSavedSession);
}
history.splice(targetIndex, 1);
writeAutoGenHistory(history);
(0, http_1.ok)(res, {
ok: true,
generation_id: generationId,
deleted_files: deletedFiles
});
}
catch (error) {
next(error);
}
});
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
try {
const body = toRecord(req.body);
@@ -1925,6 +2196,9 @@ function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIRe
const context = toRecord(body.context);
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
if (mode === "saved_user_sessions") {
throw new http_1.ApiError("AUTOGEN_MODE_NOT_SUPPORTED", "Use `/api/autoruns/autogen/save-assistant-session` to save user sessions.", 400);
}
let questions = [];
if (mode === "qwen_seed") {
if (!llmConfig) {
@@ -1963,6 +2237,7 @@ function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIRe
generation_id: generationId,
created_at: new Date().toISOString(),
mode,
title: null,
count: questions.length,
domain,
questions,
@@ -1980,7 +2255,10 @@ function buildAutoRunsRouter(openaiClient = new openaiResponsesClient_1.OpenAIRe
autogen_personality_id: toStringSafe(context.autogen_personality_id),
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
? repairAutogenMojibake(String(context.autogen_personality_prompt))
: null
: null,
source_session_id: null,
saved_session_file: null,
saved_case_set_kind: "single_turn_list"
}
: null
};
llm_normalizer/backend/dist/routes/eval.js
Vendored
+52 -8
View File
@@ -128,14 +128,23 @@ function splitQuestionCandidate(raw) {
}
return normalizeRuntimeQuestionList(chunks);
}
function normalizeRuntimeQuestions(value) {
function normalizeRuntimeQuestions(value, options) {
const raw = toArray(value)
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter((item) => item.length > 0);
if (raw.length === 0) {
return [];
}
const expanded = normalizeRuntimeQuestionList(raw.flatMap((item) => splitQuestionCandidate(item)));
const splitCandidates = options?.splitCandidates ?? true;
const expanded = splitCandidates
? normalizeRuntimeQuestionList(raw.flatMap((item) => splitQuestionCandidate(item)))
: raw
.map((item) => normalizeQuestionChunk(item))
.filter((item) => Boolean(item));
const dedupe = options?.dedupe ?? true;
if (!dedupe) {
return expanded;
}
const deduped = [];
const seen = new Set();
for (const item of expanded) {
@@ -272,6 +281,37 @@ function writeRuntimeAssistantSuiteFromQuestions(jobId, questions) {
fs_1.default.writeFileSync(path_1.default.resolve(config_1.EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
return fileName;
}
function writeRuntimeAssistantScenarioSuiteFromQuestions(jobId, questions, title) {
if (!fs_1.default.existsSync(config_1.EVAL_CASES_DIR)) {
fs_1.default.mkdirSync(config_1.EVAL_CASES_DIR, { recursive: true });
}
const turns = questions.map((question) => ({
user_message: question
}));
const payload = {
suite_id: `assistant_saved_session_runtime_${jobId}`,
suite_version: "0.1.0",
schema_version: "assistant_saved_session_runtime_v0_1",
title: typeof title === "string" ? title.trim() || null : null,
scenario_count: turns.length > 0 ? 1 : 0,
case_ids: turns.length > 0 ? ["SAVED-001"] : [],
cases: turns.length > 0
? [
{
case_id: "SAVED-001",
scenario_tag: "saved_user_sessions_runtime",
title: typeof title === "string" ? title.trim() || null : null,
question_type: turns.length > 1 ? "followup" : "direct",
broadness_level: "medium",
turns
}
]
: []
};
const fileName = `assistant_saved_session_runtime_${jobId}.json`;
fs_1.default.writeFileSync(path_1.default.resolve(config_1.EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
return fileName;
}
function readSessionConversation(runId, caseId) {
const sessionId = `${runId}-${caseId}`;
const filePath = path_1.default.resolve(config_1.ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
@@ -414,15 +454,19 @@ function buildEvalRouter(services) {
throw new http_1.ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
}
const questions = normalizeRuntimeQuestions(body.questions);
const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
const scenarioTitle = toStringSafe(body.scenarioTitle);
const jobId = `job-${(0, nanoid_1.nanoid)(10)}`;
const runId = `assistant-stage1-${(0, nanoid_1.nanoid)(10)}`;
const runtimeCaseSetFile = questions.length > 0
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
: payload.caseSetFile
? payload.caseSetFile
: undefined;
const runtimeCaseSetFile = scenarioQuestions.length > 0
? writeRuntimeAssistantScenarioSuiteFromQuestions(jobId, scenarioQuestions, scenarioTitle ?? undefined)
: questions.length > 0
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
: payload.caseSetFile
? payload.caseSetFile
: undefined;
if (!runtimeCaseSetFile) {
throw new http_1.ApiError("ASYNC_CASESET_REQUIRED", "Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.", 400);
throw new http_1.ApiError("ASYNC_CASESET_REQUIRED", "Async assistant_stage1 run requires caseSetFile, scenarioQuestions[] or explicit questions[] payload.", 400);
}
const caseSeeds = readAssistantSuiteCaseSeeds(runtimeCaseSetFile);
if (caseSeeds.length === 0) {
llm_normalizer/backend/dist/server.js
Vendored
+1 -1
View File
@@ -64,7 +64,7 @@ function createApp() {
app.use((0, normalize_1.buildNormalizeRouter)(services));
app.use((0, eval_1.buildEvalRouter)(services));
app.use((0, assistant_1.buildAssistantRouter)(services));
app.use((0, autoRuns_1.buildAutoRunsRouter)(openaiClient));
app.use((0, autoRuns_1.buildAutoRunsRouter)(services, openaiClient));
app.use((0, history_1.buildHistoryRouter)());
app.use((0, presets_1.buildPresetsRouter)());
app.use((0, accountingAgent_1.buildAccountingAgentRouter)(services));
llm_normalizer/backend/src/routes/autoRuns.ts
+349 -13
View File
@@ -11,13 +11,14 @@ import {
MANUAL_CASE_DECISION_SCHEMA_FILE,
REPORTS_DIR
} from "../config";
import type { AppServices } from "../serverContext";
import { ApiError, ok } from "../utils/http";
import { loadCapabilitiesRegistry, resolveNearestCapabilityGroup, type CapabilityGroup } from "../services/capabilitiesRegistry";
import { OpenAIResponsesClient } from "../services/openaiResponsesClient";
type AutoRunTarget = "normalizer" | "assistant_stage1" | "assistant_stage2" | "assistant_p0" | "unknown";
type AutoRunTrend = "up" | "down" | "flat";
type AutoGenMode = "qwen_seed" | "codex_creative";
type AutoGenMode = "qwen_seed" | "codex_creative" | "saved_user_sessions";
type ManualCaseDecision =
| "covered_ok"
| "covered_but_bad_answer"
@@ -175,6 +176,7 @@ interface AutoGenHistoryRecord {
generation_id: string;
created_at: string;
mode: AutoGenMode;
title: string | null;
count: number;
domain: string | null;
questions: string[];
@@ -188,6 +190,9 @@ interface AutoGenHistoryRecord {
prompt_fingerprint: string | null;
autogen_personality_id: string | null;
autogen_personality_prompt: string | null;
source_session_id?: string | null;
saved_session_file?: string | null;
saved_case_set_kind?: string | null;
} | null;
}
@@ -269,6 +274,18 @@ function clampInt(value: number | null, min: number, max: number, fallback: numb
return rounded;
}
function isAutoGenMode(value: unknown): value is AutoGenMode {
return value === "qwen_seed" || value === "codex_creative" || value === "saved_user_sessions";
}
function parseAutoGenTitle(value: unknown): string | null {
const title = toStringSafe(value);
if (!title) {
return null;
}
return title.slice(0, 160);
}
function parseManualCaseDecision(value: unknown, fallback: ManualCaseDecision = "needs_dialog_policy_fix"): ManualCaseDecision {
const normalized = toStringSafe(value);
if (!normalized) return fallback;
@@ -326,15 +343,11 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
.map((item) => ({
generation_id: toStringSafe(item.generation_id) ?? "",
created_at: toStringSafe(item.created_at) ?? new Date().toISOString(),
mode: (toStringSafe(item.mode) as AutoGenMode | null) ?? "codex_creative",
mode: isAutoGenMode(toStringSafe(item.mode)) ? (toStringSafe(item.mode) as AutoGenMode) : "codex_creative",
title: parseAutoGenTitle(item.title),
count: clampInt(toNumberSafe(item.count), 1, 300, 20),
domain: toStringSafe(item.domain),
questions: toArray(item.questions)
.map((q) => toStringSafe(q))
.filter((q): q is string => q !== null)
.map((q) => sanitizeGeneratedQuestion(q))
.filter((q) => q.length > 0)
.slice(0, 500),
questions: parseAssistantSessionQuestions(item.questions),
generated_by: toStringSafe(item.generated_by),
saved_case_set_file: toStringSafe(item.saved_case_set_file),
context: toRecord(item.context)
@@ -349,7 +362,10 @@ function readAutoGenHistory(): AutoGenHistoryRecord[] {
autogen_personality_id: toStringSafe(toRecord(item.context)?.autogen_personality_id),
autogen_personality_prompt: toStringSafe(toRecord(item.context)?.autogen_personality_prompt)
? repairAutogenMojibake(String(toRecord(item.context)?.autogen_personality_prompt))
: null
: null,
source_session_id: toStringSafe(toRecord(item.context)?.source_session_id),
saved_session_file: toStringSafe(toRecord(item.context)?.saved_session_file),
saved_case_set_kind: toStringSafe(toRecord(item.context)?.saved_case_set_kind)
}
: null
}))
@@ -1314,7 +1330,7 @@ function parseDecisionFilter(value: unknown): ManualCaseDecision | "all" {
function parseAutoGenMode(value: unknown): AutoGenMode {
const normalized = toStringSafe(value)?.toLowerCase() ?? "";
if (normalized === "qwen_seed" || normalized === "codex_creative") {
if (normalized === "qwen_seed" || normalized === "codex_creative" || normalized === "saved_user_sessions") {
return normalized;
}
return "codex_creative";
@@ -1416,6 +1432,13 @@ function sanitizeGeneratedQuestion(value: string): string {
.trim();
}
function parseAssistantSessionQuestions(value: unknown): string[] {
return toArray(value)
.map((item) => sanitizeGeneratedQuestion(typeof item === "string" ? item : ""))
.filter((item) => item.length > 0)
.slice(0, 500);
}
const AUTOGEN_QUESTION_PLACEHOLDER_PATTERN = /^(?:questions?|вопросы?|список\s+вопросов)$/iu;
const AUTOGEN_QUESTION_TAIL_PATTERNS: RegExp[] = [
/^(?:без\s+воды|по\s+факту|и\s+коротко|коротко|прям(?:\s+)?сейчас|за\s+весь\s+период|по\s+делу)\??$/iu
@@ -1723,6 +1746,19 @@ function buildAutogenCaseSetFileName(mode: AutoGenMode, generationId: string): s
return `assistant_autogen_${mode}_${stamp}_${generationId}.json`;
}
function buildSavedAssistantSessionSnapshotFileName(generationId: string): string {
const now = new Date();
const stamp = [
now.getUTCFullYear(),
String(now.getUTCMonth() + 1).padStart(2, "0"),
String(now.getUTCDate()).padStart(2, "0"),
String(now.getUTCHours()).padStart(2, "0"),
String(now.getUTCMinutes()).padStart(2, "0"),
String(now.getUTCSeconds()).padStart(2, "0")
].join("");
return `assistant_saved_session_${stamp}_${generationId}.json`;
}
function buildAutogenCaseSetPayload(input: {
generationId: string;
mode: AutoGenMode;
@@ -1757,6 +1793,118 @@ function buildAutogenCaseSetPayload(input: {
};
}
function buildSavedSessionCaseSetPayload(input: {
generationId: string;
title: string | null;
questions: string[];
}): Record<string, unknown> {
const questions = parseAssistantSessionQuestions(input.questions);
const turns = questions.map((question) => ({
user_message: question
}));
const caseId = "SAVED-001";
return {
suite_id: `assistant_saved_session_${input.generationId}`,
suite_version: "0.1.0",
schema_version: "assistant_saved_session_suite_v0_1",
generated_at: new Date().toISOString(),
generation_id: input.generationId,
mode: "saved_user_sessions",
title: input.title,
scenario_count: turns.length > 0 ? 1 : 0,
case_ids: turns.length > 0 ? [caseId] : [],
cases:
turns.length > 0
? [
{
case_id: caseId,
scenario_tag: "saved_user_sessions",
title: input.title,
question_type: turns.length > 1 ? "followup" : "direct",
broadness_level: "medium",
turns
}
]
: []
};
}
function ensureDirSync(targetDir: string): void {
if (!fs.existsSync(targetDir)) {
fs.mkdirSync(targetDir, { recursive: true });
}
}
function writeJsonFile(targetPath: string, payload: unknown): void {
ensureDirSync(path.dirname(targetPath));
fs.writeFileSync(targetPath, JSON.stringify(payload, null, 2), "utf-8");
}
function rewriteAutoGenCaseSetFile(record: AutoGenHistoryRecord): string | null {
const caseSetFile = toStringSafe(record.saved_case_set_file);
if (!caseSetFile) {
return null;
}
const targetPath = path.resolve(EVAL_CASES_DIR, caseSetFile);
const payload =
record.mode === "saved_user_sessions"
? buildSavedSessionCaseSetPayload({
generationId: record.generation_id,
title: record.title,
questions: record.questions
})
: buildAutogenCaseSetPayload({
generationId: record.generation_id,
mode: record.mode,
domain: record.domain,
questions: record.questions
});
writeJsonFile(targetPath, payload);
return caseSetFile;
}
function writeSavedAssistantSessionSnapshot(input: {
generationId: string;
sessionId: string;
title: string | null;
session: Record<string, unknown>;
questions: string[];
}): string {
const fileName = buildSavedAssistantSessionSnapshotFileName(input.generationId);
const targetPath = path.resolve(path.dirname(AUTORUN_GENERATOR_HISTORY_FILE), "saved_sessions", fileName);
writeJsonFile(targetPath, {
saved_at: new Date().toISOString(),
generation_id: input.generationId,
mode: "saved_user_sessions",
title: input.title,
source_session_id: input.sessionId,
questions: input.questions,
session: input.session
});
return fileName;
}
function resolveFileInsideDir(baseDir: string, fileName: string | null | undefined): string | null {
const normalized = toStringSafe(fileName);
if (!normalized) {
return null;
}
const targetPath = path.resolve(baseDir, normalized);
const relative = path.relative(baseDir, targetPath);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
return null;
}
return targetPath;
}
function safeDeleteFile(targetPath: string | null): string | null {
if (!targetPath || !fs.existsSync(targetPath)) {
return null;
}
fs.unlinkSync(targetPath);
return targetPath;
}
function collectPostAnalysis(
annotations: AutoRunAnnotationRecord[],
runMap: Map<string, IndexedRun>,
@@ -1854,7 +2002,7 @@ function collectPostAnalysis(
};
}
export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()): Router {
export function buildAutoRunsRouter(services: AppServices, openaiClient = new OpenAIResponsesClient()): Router {
const router = Router();
router.get("/api/autoruns/history", (req, res) => {
@@ -2251,7 +2399,7 @@ export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()):
try {
const limit = clampInt(toNumberSafe((req.query as Record<string, unknown>).limit), 1, 500, 120);
const rawMode = toStringSafe((req.query as Record<string, unknown>).mode);
const includeAllModes = !rawMode || !["qwen_seed", "codex_creative"].includes(rawMode);
const includeAllModes = !rawMode || !isAutoGenMode(rawMode);
const modeFilter = (rawMode as AutoGenMode | null) ?? "codex_creative";
const items = readAutoGenHistory()
.filter((item) => (includeAllModes ? true : item.mode === modeFilter))
@@ -2279,6 +2427,182 @@ export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()):
}
});
router.post("/api/autoruns/autogen/save-assistant-session", (req, res, next) => {
try {
const body = toRecord(req.body);
if (!body) {
throw new ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "JSON body is required", 400);
}
const sessionId = toStringSafe(body.session_id);
const title = parseAutoGenTitle(body.title);
const generatedBy = parseAnnotationAuthor(body.generated_by);
const context = toRecord(body.context);
if (!sessionId) {
throw new ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "session_id is required", 400);
}
if (!title) {
throw new ApiError("INVALID_AUTOGEN_SAVE_SESSION_PAYLOAD", "title is required", 400);
}
const session = services.assistantService.getSession(sessionId);
if (!session) {
throw new ApiError("ASSISTANT_SESSION_NOT_FOUND", `Session not found: ${sessionId}`, 404);
}
const questions = session.items
.filter((item: { role: string }) => item.role === "user")
.map((item: { text: string }) => sanitizeGeneratedQuestion(item.text))
.filter((item: string) => item.length > 0);
if (questions.length === 0) {
throw new ApiError("ASSISTANT_SESSION_EMPTY", "Assistant session has no user questions to save.", 400);
}
const generationId = generateAutogenId();
const caseSetFile = buildAutogenCaseSetFileName("saved_user_sessions", generationId);
const caseSetPath = path.resolve(EVAL_CASES_DIR, caseSetFile);
writeJsonFile(
caseSetPath,
buildSavedSessionCaseSetPayload({
generationId,
title,
questions
})
);
const snapshotFile = writeSavedAssistantSessionSnapshot({
generationId,
sessionId,
title,
session: session as unknown as Record<string, unknown>,
questions
});
const record: AutoGenHistoryRecord = {
generation_id: generationId,
created_at: new Date().toISOString(),
mode: "saved_user_sessions",
title,
count: questions.length,
domain: null,
questions,
generated_by: generatedBy,
saved_case_set_file: caseSetFile,
context: {
llm_provider: toStringSafe(context?.llm_provider),
model: toStringSafe(context?.model),
assistant_prompt_version: toStringSafe(context?.assistant_prompt_version),
decomposition_prompt_version: toStringSafe(context?.decomposition_prompt_version),
prompt_fingerprint: toStringSafe(context?.prompt_fingerprint)
? repairAutogenMojibake(String(context?.prompt_fingerprint))
: null,
autogen_personality_id: null,
autogen_personality_prompt: null,
source_session_id: sessionId,
saved_session_file: snapshotFile,
saved_case_set_kind: "assistant_session_scenario"
}
};
const history = readAutoGenHistory();
history.unshift(record);
writeAutoGenHistory(history.slice(0, 500));
ok(res, {
ok: true,
generation: record
});
} catch (error) {
next(error);
}
});
router.patch("/api/autoruns/autogen/history/:generation_id/questions", (req, res, next) => {
try {
const generationId = toStringSafe(req.params.generation_id);
const body = toRecord(req.body);
if (!generationId) {
throw new ApiError("INVALID_AUTOGEN_GENERATION_ID", "generation_id is required", 400);
}
if (!body) {
throw new ApiError("INVALID_AUTOGEN_QUESTIONS_PAYLOAD", "JSON body is required", 400);
}
const questions = parseAssistantSessionQuestions(body.questions);
if (questions.length === 0) {
throw new ApiError("INVALID_AUTOGEN_QUESTIONS_PAYLOAD", "questions must contain at least one item", 400);
}
const history = readAutoGenHistory();
const targetIndex = history.findIndex((item) => item.generation_id === generationId);
if (targetIndex < 0) {
throw new ApiError("AUTOGEN_GENERATION_NOT_FOUND", `Generation not found: ${generationId}`, 404);
}
const current = history[targetIndex];
const updated: AutoGenHistoryRecord = {
...current,
count: questions.length,
questions
};
rewriteAutoGenCaseSetFile(updated);
history[targetIndex] = updated;
writeAutoGenHistory(history);
ok(res, {
ok: true,
generation: updated
});
} catch (error) {
next(error);
}
});
router.delete("/api/autoruns/autogen/history/:generation_id", (req, res, next) => {
try {
const generationId = toStringSafe(req.params.generation_id);
if (!generationId) {
throw new ApiError("INVALID_AUTOGEN_GENERATION_ID", "generation_id is required", 400);
}
const history = readAutoGenHistory();
const targetIndex = history.findIndex((item) => item.generation_id === generationId);
if (targetIndex < 0) {
throw new ApiError("AUTOGEN_GENERATION_NOT_FOUND", `Generation not found: ${generationId}`, 404);
}
const target = history[targetIndex];
const deletedFiles: string[] = [];
const caseSetPath = resolveFileInsideDir(EVAL_CASES_DIR, target.saved_case_set_file);
const savedSessionPath = resolveFileInsideDir(
path.resolve(path.dirname(AUTORUN_GENERATOR_HISTORY_FILE), "saved_sessions"),
target.context?.saved_session_file ?? null
);
const deletedCaseSet = safeDeleteFile(caseSetPath);
if (deletedCaseSet) {
deletedFiles.push(deletedCaseSet);
}
const deletedSavedSession = safeDeleteFile(savedSessionPath);
if (deletedSavedSession) {
deletedFiles.push(deletedSavedSession);
}
history.splice(targetIndex, 1);
writeAutoGenHistory(history);
ok(res, {
ok: true,
generation_id: generationId,
deleted_files: deletedFiles
});
} catch (error) {
next(error);
}
});
router.post("/api/autoruns/autogen/generate", async (req, res, next) => {
try {
const body = toRecord(req.body);
@@ -2294,6 +2618,14 @@ export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()):
const llmConfig = parseAutogenLlmRuntimeConfig(body, context);
const personalityPrompt = toStringSafe(context?.autogen_personality_prompt);
if (mode === "saved_user_sessions") {
throw new ApiError(
"AUTOGEN_MODE_NOT_SUPPORTED",
"Use `/api/autoruns/autogen/save-assistant-session` to save user sessions.",
400
);
}
let questions: string[] = [];
if (mode === "qwen_seed") {
if (!llmConfig) {
@@ -2340,6 +2672,7 @@ export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()):
generation_id: generationId,
created_at: new Date().toISOString(),
mode,
title: null,
count: questions.length,
domain,
questions,
@@ -2357,7 +2690,10 @@ export function buildAutoRunsRouter(openaiClient = new OpenAIResponsesClient()):
autogen_personality_id: toStringSafe(context.autogen_personality_id),
autogen_personality_prompt: toStringSafe(context.autogen_personality_prompt)
? repairAutogenMojibake(String(context.autogen_personality_prompt))
: null
: null,
source_session_id: null,
saved_session_file: null,
saved_case_set_kind: "single_turn_list"
}
: null
};
llm_normalizer/backend/src/routes/eval.ts
+50 -4
View File
@@ -176,7 +176,7 @@ function splitQuestionCandidate(raw: string): string[] {
return normalizeRuntimeQuestionList(chunks);
}
function normalizeRuntimeQuestions(value: unknown): string[] {
function normalizeRuntimeQuestions(value: unknown, options?: { dedupe?: boolean; splitCandidates?: boolean }): string[] {
const raw = toArray(value)
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter((item) => item.length > 0);
@@ -184,7 +184,16 @@ function normalizeRuntimeQuestions(value: unknown): string[] {
return [];
}
const expanded = normalizeRuntimeQuestionList(raw.flatMap((item) => splitQuestionCandidate(item)));
const splitCandidates = options?.splitCandidates ?? true;
const expanded = splitCandidates
? normalizeRuntimeQuestionList(raw.flatMap((item) => splitQuestionCandidate(item)))
: raw
.map((item) => normalizeQuestionChunk(item))
.filter((item): item is string => Boolean(item));
const dedupe = options?.dedupe ?? true;
if (!dedupe) {
return expanded;
}
const deduped: string[] = [];
const seen = new Set<string>();
for (const item of expanded) {
@@ -342,6 +351,39 @@ function writeRuntimeAssistantSuiteFromQuestions(jobId: string, questions: strin
return fileName;
}
function writeRuntimeAssistantScenarioSuiteFromQuestions(jobId: string, questions: string[], title?: string): string {
if (!fs.existsSync(EVAL_CASES_DIR)) {
fs.mkdirSync(EVAL_CASES_DIR, { recursive: true });
}
const turns = questions.map((question) => ({
user_message: question
}));
const payload = {
suite_id: `assistant_saved_session_runtime_${jobId}`,
suite_version: "0.1.0",
schema_version: "assistant_saved_session_runtime_v0_1",
title: typeof title === "string" ? title.trim() || null : null,
scenario_count: turns.length > 0 ? 1 : 0,
case_ids: turns.length > 0 ? ["SAVED-001"] : [],
cases:
turns.length > 0
? [
{
case_id: "SAVED-001",
scenario_tag: "saved_user_sessions_runtime",
title: typeof title === "string" ? title.trim() || null : null,
question_type: turns.length > 1 ? "followup" : "direct",
broadness_level: "medium",
turns
}
]
: []
};
const fileName = `assistant_saved_session_runtime_${jobId}.json`;
fs.writeFileSync(path.resolve(EVAL_CASES_DIR, fileName), JSON.stringify(payload, null, 2), "utf-8");
return fileName;
}
function readSessionConversation(runId: string, caseId: string): EvalAsyncCaseInfo["messages"] {
const sessionId = `${runId}-${caseId}`;
const filePath = path.resolve(ASSISTANT_SESSIONS_DIR, `${sessionId}.json`);
@@ -489,11 +531,15 @@ export function buildEvalRouter(services: AppServices): Router {
throw new ApiError("UNSUPPORTED_ASYNC_EVAL_TARGET", "Async eval currently supports assistant_stage1 only.", 400);
}
const questions = normalizeRuntimeQuestions(body.questions);
const scenarioQuestions = normalizeRuntimeQuestions(body.scenarioQuestions, { dedupe: false, splitCandidates: false });
const scenarioTitle = toStringSafe(body.scenarioTitle);
const jobId = `job-${nanoid(10)}`;
const runId = `assistant-stage1-${nanoid(10)}`;
const runtimeCaseSetFile =
questions.length > 0
scenarioQuestions.length > 0
? writeRuntimeAssistantScenarioSuiteFromQuestions(jobId, scenarioQuestions, scenarioTitle ?? undefined)
: questions.length > 0
? writeRuntimeAssistantSuiteFromQuestions(jobId, questions)
: payload.caseSetFile
? payload.caseSetFile
@@ -502,7 +548,7 @@ export function buildEvalRouter(services: AppServices): Router {
if (!runtimeCaseSetFile) {
throw new ApiError(
"ASYNC_CASESET_REQUIRED",
"Async assistant_stage1 run requires caseSetFile or explicit questions[] payload.",
"Async assistant_stage1 run requires caseSetFile, scenarioQuestions[] or explicit questions[] payload.",
400
);
}
llm_normalizer/backend/src/server.ts
+1 -1
View File
@@ -76,7 +76,7 @@ export function createApp(): express.Express {
app.use(buildNormalizeRouter(services));
app.use(buildEvalRouter(services));
app.use(buildAssistantRouter(services));
app.use(buildAutoRunsRouter(openaiClient));
app.use(buildAutoRunsRouter(services, openaiClient));
app.use(buildHistoryRouter());
app.use(buildPresetsRouter());
app.use(buildAccountingAgentRouter(services));