Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага

2026-03-28 02:17:19 +03:00
parent 914843a8ba
commit a06e575be4
367 changed files with 432257 additions and 3627 deletions
@@ -12,6 +12,7 @@ import {
  FEATURE_ASSISTANT_STAGE2_EVAL_V1,
  REPORTS_DIR
 } from "../config";
+import { P0EvalRunner } from "../eval/p0_eval_runner";
 import type { AssistantMessageResponsePayload } from "../types/assistant";
 import type {
  EvalTarget,
@@ -328,6 +329,98 @@ const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_compar
 const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
 const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
 const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
+const INMEM_EVAL_REPORT_PREFIX = "inmem_eval_report:";
+const INMEM_EVAL_REPORTS = new Map<string, Record<string, unknown>>();
+
+function isNoSpaceError(error: unknown): boolean {
+  const code = (error as { code?: unknown } | null)?.code;
+  return code === "ENOSPC";
+}
+
+function tryWriteJsonFile(pathname: string, value: unknown): boolean {
+  try {
+    writeJsonFile(pathname, value);
+    return true;
+  } catch (error) {
+    if (isNoSpaceError(error)) {
+      return false;
+    }
+    throw error;
+  }
+}
+
+function tryWriteTextFile(pathname: string, value: string): boolean {
+  try {
+    fs.writeFileSync(pathname, value, "utf-8");
+    return true;
+  } catch (error) {
+    if (isNoSpaceError(error)) {
+      return false;
+    }
+    throw error;
+  }
+}
+
+function putInMemoryEvalReport(report: Record<string, unknown>): string {
+  const key = `${INMEM_EVAL_REPORT_PREFIX}${nanoid(12)}`;
+  INMEM_EVAL_REPORTS.set(key, report);
+  return key;
+}
+
+function readEvalReportByRef(ref: string): { report: Record<string, unknown>; resolved_path: string } {
+  if (ref.startsWith(INMEM_EVAL_REPORT_PREFIX)) {
+    const report = INMEM_EVAL_REPORTS.get(ref);
+    if (!report) {
+      throw new Error(`In-memory eval report not found: ${ref}`);
+    }
+    return {
+      report,
+      resolved_path: ref
+    };
+  }
+
+  const resolvedPath = resolveReadablePath(ref);
+  const report = JSON.parse(fs.readFileSync(resolvedPath, "utf-8")) as Record<string, unknown>;
+  return {
+    report,
+    resolved_path: resolvedPath
+  };
+}
+
+function compactAssistantStage1Report(report: Record<string, unknown>): Record<string, unknown> {
+  const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
+  const compactResults = results.map((item) => ({
+    case_id: item.case_id ?? null,
+    scenario_tag: item.scenario_tag ?? null,
+    accountant_usefulness_score: item.accountant_usefulness_score ?? null,
+    accountant_metrics:
+      typeof item.accountant_metrics === "object" && item.accountant_metrics !== null ? item.accountant_metrics : null
+  }));
+  return {
+    ...report,
+    results: compactResults
+  };
+}
+
+function compactAssistantStage2Report(report: Record<string, unknown>): Record<string, unknown> {
+  const results = Array.isArray(report.results) ? (report.results as Array<Record<string, unknown>>) : [];
+  const compactResults = results.map((item) => {
+    const metricSubscores = (item.metric_subscores ?? {}) as Record<string, unknown>;
+    return {
+      case_id: item.case_id ?? null,
+      metric_subscores: {
+        problem_clarity_score: metricSubscores.problem_clarity_score ?? null,
+        mechanism_coherence_score: metricSubscores.mechanism_coherence_score ?? null,
+        problem_first_answer_rate: metricSubscores.problem_first_answer_rate ?? null,
+        entity_leakage_rate: metricSubscores.entity_leakage_rate ?? null
+      }
+    };
+  });
+  return {
+    ...report,
+    results: compactResults
+  };
+}

 type AssistantMetricKey = keyof AssistantEvalMetricVector;
 type AssistantStage2MetricKey = keyof AssistantStage2MetricVector;
@@ -1140,7 +1233,7 @@ export class EvalService {
    };

    ensureDir(EVAL_CASES_DIR);
-    writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
+    tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);

    return report;
  }
@@ -1534,8 +1627,9 @@ export class EvalService {
    currentReport: Record<string, unknown>;
    baselineReportFile: string;
  }): Record<string, unknown> {
-    const baselinePath = resolveReadablePath(input.baselineReportFile);
-    const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
+    const baselineRef = readEvalReportByRef(input.baselineReportFile);
+    const baselinePath = baselineRef.resolved_path;
+    const baselineReport = baselineRef.report;
    const currentReport = input.currentReport;
    const metricKeys: AssistantMetricKey[] = [
      "retrieval_differentiation_rate",
@@ -1633,14 +1727,15 @@ export class EvalService {
    ensureDir(REPORTS_DIR);
    const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
    const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
-    writeJsonFile(jsonPath, comparisonReport);
-    fs.writeFileSync(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport), "utf-8");
+    const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
+    const mdWritten = tryWriteTextFile(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport));
+    const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);

    return {
      ...comparisonReport,
      artifacts: {
-        comparison_report_json_path: jsonPath,
-        comparison_report_md_path: mdPath
+        comparison_report_json_path: comparisonRef,
+        comparison_report_md_path: mdWritten ? mdPath : null
      }
    };
  }
@@ -1649,8 +1744,9 @@ export class EvalService {
    currentReport: Record<string, unknown>;
    baselineReportFile: string;
  }): Record<string, unknown> {
-    const baselinePath = resolveReadablePath(input.baselineReportFile);
-    const baselineReport = JSON.parse(fs.readFileSync(baselinePath, "utf-8")) as Record<string, unknown>;
+    const baselineRef = readEvalReportByRef(input.baselineReportFile);
+    const baselinePath = baselineRef.resolved_path;
+    const baselineReport = baselineRef.report;
    const currentReport = input.currentReport;
    const metricKeys: AssistantStage2MetricKey[] = [
      "problem_unit_precision",
@@ -1760,14 +1856,15 @@ export class EvalService {
    ensureDir(REPORTS_DIR);
    const jsonPath = path.resolve(REPORTS_DIR, `${comparisonId}.json`);
    const mdPath = path.resolve(REPORTS_DIR, `${comparisonId}.md`);
-    writeJsonFile(jsonPath, comparisonReport);
-    fs.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
+    const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
+    const mdWritten = tryWriteTextFile(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport));
+    const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);

    return {
      ...comparisonReport,
      artifacts: {
-        comparison_report_json_path: jsonPath,
-        comparison_report_md_path: mdPath
+        comparison_report_json_path: comparisonRef,
+        comparison_report_md_path: mdWritten ? mdPath : null
      }
    };
  }
@@ -1803,7 +1900,7 @@ export class EvalService {

      try {
        for (const turn of suiteCase.turns) {
-          const response = await assistantService.handleMessage({
+          const response = (await assistantService.handleMessage({
            session_id: sessionId,
            user_message: turn.user_message,
            message: turn.user_message,
@@ -1819,7 +1916,7 @@ export class EvalService {
            domainPrompt: payload.normalizeConfig.domainPrompt,
            fewShotExamples: payload.normalizeConfig.fewShotExamples,
            useMock: payload.useMock
-          });
+          })) as AssistantMessageResponsePayload;
          turnResponses.push(response);
          requestsTotal += 1;
        }
@@ -2099,12 +2196,14 @@ export class EvalService {
    ensureDir(REPORTS_DIR);
    const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
    const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
-    writeJsonFile(runJsonPath, report);
-    fs.writeFileSync(runMdPath, buildAssistantEvalMarkdownReport(report), "utf-8");
+    const compactReport = compactAssistantStage1Report(report);
+    const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
+    const mdWritten = tryWriteTextFile(runMdPath, buildAssistantEvalMarkdownReport(compactReport));
+    const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);

    report.artifacts = {
-      run_report_json_path: runJsonPath,
-      run_report_md_path: runMdPath
+      run_report_json_path: runReportRef,
+      run_report_md_path: mdWritten ? runMdPath : null
    };

    if (payload.compareWithReportFile) {
@@ -2151,7 +2250,7 @@ export class EvalService {

      try {
        for (const turn of suiteCase.turns) {
-          const response = await assistantService.handleMessage({
+          const response = (await assistantService.handleMessage({
            session_id: sessionId,
            user_message: turn.user_message,
            message: turn.user_message,
@@ -2167,7 +2266,7 @@ export class EvalService {
            domainPrompt: payload.normalizeConfig.domainPrompt,
            fewShotExamples: payload.normalizeConfig.fewShotExamples,
            useMock: payload.useMock
-          });
+          })) as AssistantMessageResponsePayload;
          turnResponses.push(response);
          requestsTotal += 1;
        }
@@ -2393,12 +2492,14 @@ export class EvalService {
    ensureDir(REPORTS_DIR);
    const runJsonPath = path.resolve(REPORTS_DIR, `${runId}.json`);
    const runMdPath = path.resolve(REPORTS_DIR, `${runId}.md`);
-    writeJsonFile(runJsonPath, report);
-    fs.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
+    const compactReport = compactAssistantStage2Report(report);
+    const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
+    const mdWritten = tryWriteTextFile(runMdPath, buildAssistantStage2EvalMarkdownReport(compactReport));
+    const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);

    report.artifacts = {
-      run_report_json_path: runJsonPath,
-      run_report_md_path: runMdPath
+      run_report_json_path: runReportRef,
+      run_report_md_path: mdWritten ? runMdPath : null
    };

    if (payload.compareWithReportFile) {
@@ -2411,6 +2512,33 @@ export class EvalService {
    return report;
  }

+  private async runAssistantP0(payload: {
+    normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
+    caseIds?: string[];
+    useMock?: boolean;
+    mode: EvalRunMode;
+    caseSetFile?: string;
+    compareWithReportFile?: string;
+  }): Promise<Record<string, unknown>> {
+    if (!FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
+      throw new ApiError(
+        "ASSISTANT_P0_EVAL_DISABLED",
+        "Assistant P0 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.",
+        409
+      );
+    }
+
+    const runner = new P0EvalRunner(this.normalizerService);
+    return runner.run({
+      normalizeConfig: payload.normalizeConfig,
+      caseIds: payload.caseIds,
+      useMock: payload.useMock,
+      mode: payload.mode,
+      caseSetFile: payload.caseSetFile,
+      compareWithReportFile: payload.compareWithReportFile
+    });
+  }
+
  public async run(payload: {
    normalizeConfig: Omit<NormalizeRequestPayload, "userQuestion" | "context">;
    caseIds?: string[];
@@ -2446,6 +2574,17 @@ export class EvalService {
      });
    }

+    if (evalTarget === "assistant_p0") {
+      return this.runAssistantP0({
+        normalizeConfig: payload.normalizeConfig,
+        caseIds: payload.caseIds,
+        useMock: payload.useMock,
+        mode,
+        caseSetFile: payload.caseSetFile,
+        compareWithReportFile: payload.compareWithReportFile
+      });
+    }
+
    const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
    const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
    const isV2 =
@@ -2659,7 +2798,7 @@ export class EvalService {
    };

    ensureDir(EVAL_CASES_DIR);
-    writeJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);
+    tryWriteJsonFile(path.resolve(EVAL_CASES_DIR, `${runId}.report.json`), report);

    const shouldWriteV11Artifacts =
      mode === "single-pass-strict" &&
@@ -2668,14 +2807,13 @@ export class EvalService {

    if (shouldWriteV11Artifacts) {
      ensureDir(REPORTS_DIR);
-      writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
-      fs.writeFileSync(
+      tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
+      tryWriteTextFile(
        path.resolve(REPORTS_DIR, "normalizer_eval_v1_1_run.md"),
        buildMarkdownReport({
          ...report,
          report_title: "LLM Normalizer v1.1 Eval Run"
-        }),
-        "utf-8"
+        })
      );
    }

@@ -2687,14 +2825,13 @@ export class EvalService {

    if (shouldWriteV1121EvalArtifacts) {
      ensureDir(REPORTS_DIR);
-      writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
-      fs.writeFileSync(
+      tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
+      tryWriteTextFile(
        path.resolve(REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"),
        buildMarkdownReport({
          ...report,
          report_title: "LLM Normalizer v1.1.2.1 Eval Run"
-        }),
-        "utf-8"
+        })
      );
    }

@@ -2705,14 +2842,13 @@ export class EvalService {

    if (shouldWriteV111MicroArtifacts) {
      ensureDir(REPORTS_DIR);
-      writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
-      fs.writeFileSync(
+      tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
+      tryWriteTextFile(
        path.resolve(REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"),
        buildMarkdownReport({
          ...report,
          report_title: "LLM Normalizer v1.1.1 Micro Eval"
-        }),
-        "utf-8"
+        })
      );
    }

@@ -2723,14 +2859,13 @@ export class EvalService {

    if (shouldWriteV112MicroArtifacts) {
      ensureDir(REPORTS_DIR);
-      writeJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
-      fs.writeFileSync(
+      tryWriteJsonFile(path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
+      tryWriteTextFile(
        path.resolve(REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"),
        buildMarkdownReport({
          ...report,
          report_title: "LLM Normalizer v1.1.2 Micro Eval"
-        }),
-        "utf-8"
+        })
      );
    }