Stage 3: улучшена логика жизненного цикла и очищены ответы ассистента

2026-03-26 20:21:51 +03:00
parent d0b842adb0
commit 914843a8ba
81 changed files with 18051 additions and 654 deletions
@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+
+const fs = require("node:fs");
+const path = require("node:path");
+const request = require("supertest");
+
+const STAGE3_SUITE_RELATIVE = path.join("eval_cases", "assistant_stage3_lifecycle_probe_v0_1.json");
+const FLAG_KEYS = [
+  "FEATURE_ASSISTANT_PROBLEM_UNITS_V1",
+  "FEATURE_ASSISTANT_ANSWER_POLICY_V11",
+  "FEATURE_ASSISTANT_BROAD_GUARD_V1",
+  "FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1",
+  "FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1",
+  "FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1",
+  "FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1",
+  "FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1",
+  "FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1"
+];
+
+function parseArgs(argv) {
+  const args = {
+    runDir: "",
+    suitePath: "",
+    outputSubdir: path.join("prompt_dialogs", "stage3_lifecycle_probe")
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const token = argv[i];
+    if (token === "--run-dir") {
+      args.runDir = String(argv[i + 1] ?? "");
+      i += 1;
+      continue;
+    }
+    if (token === "--suite-path") {
+      args.suitePath = String(argv[i + 1] ?? "");
+      i += 1;
+      continue;
+    }
+    if (token === "--output-subdir") {
+      args.outputSubdir = String(argv[i + 1] ?? "");
+      i += 1;
+    }
+  }
+  return args;
+}
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function writeUtf8Bom(filePath, content) {
+  fs.writeFileSync(filePath, `\uFEFF${content}`, "utf8");
+}
+
+function toSafeFileToken(value) {
+  return String(value)
+    .trim()
+    .replace(/\s+/g, "_")
+    .replace(/[^a-zA-Z0-9_-]/g, "_")
+    .replace(/_+/g, "_");
+}
+
+function readJson(filePath) {
+  const raw = fs.readFileSync(filePath, "utf8").replace(/^\uFEFF/, "");
+  return JSON.parse(raw);
+}
+
+function findLatestRunDir(runsRoot) {
+  if (!fs.existsSync(runsRoot)) {
+    throw new Error(`Runs folder not found: ${runsRoot}`);
+  }
+  const dirs = fs
+    .readdirSync(runsRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(runsRoot, entry.name))
+    .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs);
+  if (dirs.length === 0) {
+    throw new Error(`No run directories found under: ${runsRoot}`);
+  }
+  return dirs[0];
+}
+
+function resolveRunDir(args, runsRoot) {
+  if (args.runDir) {
+    return path.resolve(args.runDir);
+  }
+  return findLatestRunDir(runsRoot);
+}
+
+function setLifecycleFlags() {
+  const original = {};
+  for (const key of FLAG_KEYS) {
+    original[key] = process.env[key];
+  }
+  process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 = "1";
+  process.env.FEATURE_ASSISTANT_ANSWER_POLICY_V11 = "1";
+  process.env.FEATURE_ASSISTANT_BROAD_GUARD_V1 = "1";
+  process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 = "1";
+  process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 = "1";
+  process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 = "1";
+  process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 = "0";
+  process.env.FEATURE_ASSISTANT_LIFECYCLE_RUNTIME_V1 = "1";
+  process.env.FEATURE_ASSISTANT_LIFECYCLE_ANSWER_V1 = "1";
+  return original;
+}
+
+function restoreFlags(original) {
+  for (const key of FLAG_KEYS) {
+    const value = original[key];
+    if (value === undefined) {
+      delete process.env[key];
+    } else {
+      process.env[key] = value;
+    }
+  }
+}
+
+function summarizeDebug(debug) {
+  const routeSummary = Array.isArray(debug?.route_summary) ? debug.route_summary : [];
+  const retrievalResults = Array.isArray(debug?.retrieval_results) ? debug.retrieval_results : [];
+  const routed = retrievalResults.filter((item) => String(item?.route ?? "") !== "no_route");
+  const problemUnits = routed.reduce((acc, item) => {
+    const list = Array.isArray(item?.problem_units) ? item.problem_units : [];
+    return acc + list.length;
+  }, 0);
+  return {
+    route_summary: routeSummary,
+    routed_retrieval_count: routed.length,
+    problem_units_count: problemUnits,
+    problem_answer_mode: typeof debug?.problem_answer_mode === "string" ? debug.problem_answer_mode : ""
+  };
+}
+
+function buildMarkdown(dialog) {
+  const lines = [];
+  lines.push(`# ${dialog.case_id}`);
+  lines.push("");
+  lines.push(`- session_id: ${dialog.session_id || "n/a"}`);
+  lines.push(`- reply_type: ${dialog.reply_type || "n/a"}`);
+  lines.push(`- trace_id: ${dialog.trace_id || "n/a"}`);
+  lines.push(`- status: ${dialog.http_status}`);
+  lines.push("");
+  lines.push("## User");
+  lines.push(dialog.user_message || "");
+  lines.push("");
+  lines.push("## Assistant");
+  lines.push(dialog.assistant_reply || "");
+  lines.push("");
+  lines.push("## Debug Summary");
+  lines.push("```json");
+  lines.push(JSON.stringify(dialog.debug_summary, null, 2));
+  lines.push("```");
+  lines.push("");
+  return lines.join("\n");
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const backendRoot = path.resolve(__dirname, "..");
+  const repoRoot = path.resolve(backendRoot, "..");
+  const runsRoot = path.join(repoRoot, "docs", "runs");
+  const runDir = resolveRunDir(args, runsRoot);
+  const suitePath = args.suitePath ? path.resolve(args.suitePath) : path.join(repoRoot, STAGE3_SUITE_RELATIVE);
+  const suite = readJson(suitePath);
+  const dialogsDir = path.join(runDir, args.outputSubdir);
+  ensureDir(dialogsDir);
+  ensureDir(path.join(runDir, "prompt_dialogs"));
+
+  const originalFlags = setLifecycleFlags();
+  let app;
+  try {
+    const { createApp } = require(path.join(backendRoot, "dist", "server.js"));
+    app = createApp();
+  } finally {
+    restoreFlags(originalFlags);
+  }
+
+  const indexRows = [];
+  const generatedAt = new Date().toISOString();
+
+  for (let i = 0; i < suite.cases.length; i += 1) {
+    const probeCase = suite.cases[i];
+    const caseId = String(probeCase.case_id || `case_${i + 1}`);
+    const userMessage = String(probeCase?.turns?.[0]?.user_message || "");
+    const response = await request(app).post("/api/assistant/message").send({
+      useMock: true,
+      promptVersion: "normalizer_v2_0_2",
+      user_message: userMessage
+    });
+    const body = response.body || {};
+    const sessionId = String(body.session_id || "");
+
+    let session = null;
+    if (sessionId) {
+      const sessionResponse = await request(app).get(`/api/assistant/session/${encodeURIComponent(sessionId)}`);
+      if (sessionResponse.status === 200 && sessionResponse.body?.ok) {
+        session = sessionResponse.body.session ?? null;
+      }
+    }
+
+    const debugSummary = summarizeDebug(body.debug);
+    const artifact = {
+      schema_version: "assistant_prompt_dialog_v0_1",
+      generated_at: generatedAt,
+      suite_id: suite.suite_id,
+      case_id: caseId,
+      scenario_tag: probeCase.scenario_tag || "",
+      expected_hints: probeCase.expected_hints || {},
+      lifecycle_focus: probeCase.lifecycle_focus || {},
+      request: {
+        useMock: true,
+        promptVersion: "normalizer_v2_0_2",
+        user_message: userMessage
+      },
+      http_status: response.status,
+      session_id: sessionId,
+      trace_id: String(body.debug?.trace_id || body.conversation_item?.trace_id || ""),
+      reply_type: String(body.reply_type || ""),
+      assistant_reply: String(body.assistant_reply || ""),
+      user_message: userMessage,
+      conversation: Array.isArray(body.conversation) ? body.conversation : [],
+      conversation_item: body.conversation_item || null,
+      debug_summary: debugSummary,
+      debug: body.debug || {},
+      session
+    };
+
+    const order = String(i + 1).padStart(2, "0");
+    const fileStem = `${order}_${toSafeFileToken(caseId)}`;
+    const jsonFile = `${fileStem}.json`;
+    const mdFile = `${fileStem}.md`;
+    writeUtf8Bom(path.join(dialogsDir, jsonFile), `${JSON.stringify(artifact, null, 2)}\n`);
+    writeUtf8Bom(path.join(dialogsDir, mdFile), buildMarkdown(artifact));
+
+    indexRows.push({
+      case_id: caseId,
+      scenario_tag: String(probeCase.scenario_tag || ""),
+      reply_type: artifact.reply_type,
+      session_id: artifact.session_id,
+      trace_id: artifact.trace_id,
+      routed_retrieval_count: debugSummary.routed_retrieval_count,
+      problem_units_count: debugSummary.problem_units_count,
+      prompt_dialog_json: path.join(args.outputSubdir, jsonFile).replace(/\\/g, "/"),
+      prompt_dialog_md: path.join(args.outputSubdir, mdFile).replace(/\\/g, "/")
+    });
+  }
+
+  const indexPayload = {
+    schema_version: "assistant_prompt_dialog_index_v0_1",
+    generated_at: generatedAt,
+    run_dir: runDir,
+    suite_id: suite.suite_id,
+    scenario_count: suite.scenario_count,
+    dialogs: indexRows
+  };
+  writeUtf8Bom(path.join(runDir, "prompt_dialogs", "index.json"), `${JSON.stringify(indexPayload, null, 2)}\n`);
+
+  process.stdout.write(
+    [
+      `run_dir=${runDir}`,
+      `suite_id=${suite.suite_id}`,
+      `dialogs_generated=${indexRows.length}`,
+      `dialogs_folder=${dialogsDir}`
+    ].join("\n") + "\n"
+  );
+}
+
+main().catch((error) => {
+  process.stderr.write(`${error?.stack || error}\n`);
+  process.exit(1);
+});