Этап 4 / Волна 10: корректировка settlement-кейса — доменная фиксация синтеза, честное покрытие, удержание фокуса / Этап 4 / Волна 11: бизнес-якоря, доменное заземление и устранение утечки дебага

2026-03-28 02:17:19 +03:00
parent 914843a8ba
commit a06e575be4
367 changed files with 432257 additions and 3627 deletions
@@ -8,6 +8,7 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const nanoid_1 = require("nanoid");
 const config_1 = require("../config");
+const p0_eval_runner_1 = require("../eval/p0_eval_runner");
 const stage1Contracts_1 = require("../types/stage1Contracts");
 const stage2EvalContracts_1 = require("../types/stage2EvalContracts");
 const http_1 = require("../utils/http");
@@ -218,6 +219,91 @@ const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_compar
 const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
 const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
 const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
+const INMEM_EVAL_REPORT_PREFIX = "inmem_eval_report:";
+const INMEM_EVAL_REPORTS = new Map();
+function isNoSpaceError(error) {
+    const code = error?.code;
+    return code === "ENOSPC";
+}
+function tryWriteJsonFile(pathname, value) {
+    try {
+        (0, files_1.writeJsonFile)(pathname, value);
+        return true;
+    }
+    catch (error) {
+        if (isNoSpaceError(error)) {
+            return false;
+        }
+        throw error;
+    }
+}
+function tryWriteTextFile(pathname, value) {
+    try {
+        fs_1.default.writeFileSync(pathname, value, "utf-8");
+        return true;
+    }
+    catch (error) {
+        if (isNoSpaceError(error)) {
+            return false;
+        }
+        throw error;
+    }
+}
+function putInMemoryEvalReport(report) {
+    const key = `${INMEM_EVAL_REPORT_PREFIX}${(0, nanoid_1.nanoid)(12)}`;
+    INMEM_EVAL_REPORTS.set(key, report);
+    return key;
+}
+function readEvalReportByRef(ref) {
+    if (ref.startsWith(INMEM_EVAL_REPORT_PREFIX)) {
+        const report = INMEM_EVAL_REPORTS.get(ref);
+        if (!report) {
+            throw new Error(`In-memory eval report not found: ${ref}`);
+        }
+        return {
+            report,
+            resolved_path: ref
+        };
+    }
+    const resolvedPath = resolveReadablePath(ref);
+    const report = JSON.parse(fs_1.default.readFileSync(resolvedPath, "utf-8"));
+    return {
+        report,
+        resolved_path: resolvedPath
+    };
+}
+function compactAssistantStage1Report(report) {
+    const results = Array.isArray(report.results) ? report.results : [];
+    const compactResults = results.map((item) => ({
+        case_id: item.case_id ?? null,
+        scenario_tag: item.scenario_tag ?? null,
+        accountant_usefulness_score: item.accountant_usefulness_score ?? null,
+        accountant_metrics: typeof item.accountant_metrics === "object" && item.accountant_metrics !== null ? item.accountant_metrics : null
+    }));
+    return {
+        ...report,
+        results: compactResults
+    };
+}
+function compactAssistantStage2Report(report) {
+    const results = Array.isArray(report.results) ? report.results : [];
+    const compactResults = results.map((item) => {
+        const metricSubscores = (item.metric_subscores ?? {});
+        return {
+            case_id: item.case_id ?? null,
+            metric_subscores: {
+                problem_clarity_score: metricSubscores.problem_clarity_score ?? null,
+                mechanism_coherence_score: metricSubscores.mechanism_coherence_score ?? null,
+                problem_first_answer_rate: metricSubscores.problem_first_answer_rate ?? null,
+                entity_leakage_rate: metricSubscores.entity_leakage_rate ?? null
+            }
+        };
+    });
+    return {
+        ...report,
+        results: compactResults
+    };
+}
 const KNOWN_PROBLEM_UNIT_TYPES = [
    "document_conflict",
    "broken_chain_segment",
@@ -900,7 +986,7 @@ class EvalService {
            results
        };
        (0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
-        (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
+        tryWriteJsonFile(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
        return report;
    }
    collectAssistantSignals(finalResponse, turnResponses) {
@@ -1237,8 +1323,9 @@ class EvalService {
        };
    }
    buildAssistantComparisonReport(input) {
-        const baselinePath = resolveReadablePath(input.baselineReportFile);
-        const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
+        const baselineRef = readEvalReportByRef(input.baselineReportFile);
+        const baselinePath = baselineRef.resolved_path;
+        const baselineReport = baselineRef.report;
        const currentReport = input.currentReport;
        const metricKeys = [
            "retrieval_differentiation_rate",
@@ -1330,19 +1417,21 @@ class EvalService {
        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
        const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
        const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
-        (0, files_1.writeJsonFile)(jsonPath, comparisonReport);
-        fs_1.default.writeFileSync(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport), "utf-8");
+        const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
+        const mdWritten = tryWriteTextFile(mdPath, buildAssistantComparisonMarkdownReport(comparisonReport));
+        const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
        return {
            ...comparisonReport,
            artifacts: {
-                comparison_report_json_path: jsonPath,
-                comparison_report_md_path: mdPath
+                comparison_report_json_path: comparisonRef,
+                comparison_report_md_path: mdWritten ? mdPath : null
            }
        };
    }
    buildAssistantStage2ComparisonReport(input) {
-        const baselinePath = resolveReadablePath(input.baselineReportFile);
-        const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
+        const baselineRef = readEvalReportByRef(input.baselineReportFile);
+        const baselinePath = baselineRef.resolved_path;
+        const baselineReport = baselineRef.report;
        const currentReport = input.currentReport;
        const metricKeys = [
            "problem_unit_precision",
@@ -1446,13 +1535,14 @@ class EvalService {
        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
        const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
        const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
-        (0, files_1.writeJsonFile)(jsonPath, comparisonReport);
-        fs_1.default.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
+        const jsonWritten = tryWriteJsonFile(jsonPath, comparisonReport);
+        const mdWritten = tryWriteTextFile(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport));
+        const comparisonRef = jsonWritten ? jsonPath : putInMemoryEvalReport(comparisonReport);
        return {
            ...comparisonReport,
            artifacts: {
-                comparison_report_json_path: jsonPath,
-                comparison_report_md_path: mdPath
+                comparison_report_json_path: comparisonRef,
+                comparison_report_md_path: mdWritten ? mdPath : null
            }
        };
    }
@@ -1473,7 +1563,7 @@ class EvalService {
            const limitations = [];
            try {
                for (const turn of suiteCase.turns) {
-                    const response = await assistantService.handleMessage({
+                    const response = (await assistantService.handleMessage({
                        session_id: sessionId,
                        user_message: turn.user_message,
                        message: turn.user_message,
@@ -1489,7 +1579,7 @@ class EvalService {
                        domainPrompt: payload.normalizeConfig.domainPrompt,
                        fewShotExamples: payload.normalizeConfig.fewShotExamples,
                        useMock: payload.useMock
-                    });
+                    }));
                    turnResponses.push(response);
                    requestsTotal += 1;
                }
@@ -1773,11 +1863,13 @@ class EvalService {
        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
        const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
        const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
-        (0, files_1.writeJsonFile)(runJsonPath, report);
-        fs_1.default.writeFileSync(runMdPath, buildAssistantEvalMarkdownReport(report), "utf-8");
+        const compactReport = compactAssistantStage1Report(report);
+        const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
+        const mdWritten = tryWriteTextFile(runMdPath, buildAssistantEvalMarkdownReport(compactReport));
+        const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
        report.artifacts = {
-            run_report_json_path: runJsonPath,
-            run_report_md_path: runMdPath
+            run_report_json_path: runReportRef,
+            run_report_md_path: mdWritten ? runMdPath : null
        };
        if (payload.compareWithReportFile) {
            report.comparison = this.buildAssistantComparisonReport({
@@ -1806,7 +1898,7 @@ class EvalService {
            const expectedProblemFirst = suiteCase.expected_hints?.expected_problem_first ?? (suiteCase.broadness_level !== "low" || suiteCase.question_type !== "direct");
            try {
                for (const turn of suiteCase.turns) {
-                    const response = await assistantService.handleMessage({
+                    const response = (await assistantService.handleMessage({
                        session_id: sessionId,
                        user_message: turn.user_message,
                        message: turn.user_message,
@@ -1822,7 +1914,7 @@ class EvalService {
                        domainPrompt: payload.normalizeConfig.domainPrompt,
                        fewShotExamples: payload.normalizeConfig.fewShotExamples,
                        useMock: payload.useMock
-                    });
+                    }));
                    turnResponses.push(response);
                    requestsTotal += 1;
                }
@@ -2045,11 +2137,13 @@ class EvalService {
        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
        const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
        const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
-        (0, files_1.writeJsonFile)(runJsonPath, report);
-        fs_1.default.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
+        const compactReport = compactAssistantStage2Report(report);
+        const jsonWritten = tryWriteJsonFile(runJsonPath, compactReport);
+        const mdWritten = tryWriteTextFile(runMdPath, buildAssistantStage2EvalMarkdownReport(compactReport));
+        const runReportRef = jsonWritten ? runJsonPath : putInMemoryEvalReport(compactReport);
        report.artifacts = {
-            run_report_json_path: runJsonPath,
-            run_report_md_path: runMdPath
+            run_report_json_path: runReportRef,
+            run_report_md_path: mdWritten ? runMdPath : null
        };
        if (payload.compareWithReportFile) {
            report.comparison = this.buildAssistantStage2ComparisonReport({
@@ -2059,6 +2153,20 @@ class EvalService {
        }
        return report;
    }
+    async runAssistantP0(payload) {
+        if (!config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
+            throw new http_1.ApiError("ASSISTANT_P0_EVAL_DISABLED", "Assistant P0 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.", 409);
+        }
+        const runner = new p0_eval_runner_1.P0EvalRunner(this.normalizerService);
+        return runner.run({
+            normalizeConfig: payload.normalizeConfig,
+            caseIds: payload.caseIds,
+            useMock: payload.useMock,
+            mode: payload.mode,
+            caseSetFile: payload.caseSetFile,
+            compareWithReportFile: payload.compareWithReportFile
+        });
+    }
    async run(payload) {
        const mode = payload.mode ?? "standard";
        const evalTarget = payload.evalTarget ?? "normalizer";
@@ -2082,6 +2190,16 @@ class EvalService {
                compareWithReportFile: payload.compareWithReportFile
            });
        }
+        if (evalTarget === "assistant_p0") {
+            return this.runAssistantP0({
+                normalizeConfig: payload.normalizeConfig,
+                caseIds: payload.caseIds,
+                useMock: payload.useMock,
+                mode,
+                caseSetFile: payload.caseSetFile,
+                compareWithReportFile: payload.compareWithReportFile
+            });
+        }
        const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
        const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
        const isV2 = promptVersion.startsWith("normalizer_v2") || schemaVersion === "v2" || schemaVersion === "v2_0_1" || schemaVersion === "v2_0_2";
@@ -2269,17 +2387,17 @@ class EvalService {
            results
        };
        (0, files_1.ensureDir)(config_1.EVAL_CASES_DIR);
-        (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
+        tryWriteJsonFile(path_1.default.resolve(config_1.EVAL_CASES_DIR, `${runId}.report.json`), report);
        const shouldWriteV11Artifacts = mode === "single-pass-strict" &&
            Boolean(payload.caseSetFile) &&
            path_1.default.basename(String(payload.caseSetFile)).toLowerCase() === "normalizer_eval_v1_1_30cases.json";
        if (shouldWriteV11Artifacts) {
            (0, files_1.ensureDir)(config_1.REPORTS_DIR);
-            (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
-            fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.md"), buildMarkdownReport({
+            tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.json"), report);
+            tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_eval_v1_1_run.md"), buildMarkdownReport({
                ...report,
                report_title: "LLM Normalizer v1.1 Eval Run"
-            }), "utf-8");
+            }));
        }
        const shouldWriteV1121EvalArtifacts = mode === "single-pass-strict" &&
            String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_2_1" &&
@@ -2287,33 +2405,33 @@ class EvalService {
            path_1.default.basename(String(payload.caseSetFile)).toLowerCase() === "normalizer_eval_v1_1_2_1_30cases.json";
        if (shouldWriteV1121EvalArtifacts) {
            (0, files_1.ensureDir)(config_1.REPORTS_DIR);
-            (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
-            fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"), buildMarkdownReport({
+            tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.json"), report);
+            tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_1_eval.md"), buildMarkdownReport({
                ...report,
                report_title: "LLM Normalizer v1.1.2.1 Eval Run"
-            }), "utf-8");
+            }));
        }
        const shouldWriteV111MicroArtifacts = mode === "single-pass-strict" &&
            String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_1" &&
            isSameCaseSet(payload.caseIds, V111_MICRO_CASE_IDS);
        if (shouldWriteV111MicroArtifacts) {
            (0, files_1.ensureDir)(config_1.REPORTS_DIR);
-            (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
-            fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"), buildMarkdownReport({
+            tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.json"), report);
+            tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_1_micro_eval.md"), buildMarkdownReport({
                ...report,
                report_title: "LLM Normalizer v1.1.1 Micro Eval"
-            }), "utf-8");
+            }));
        }
        const shouldWriteV112MicroArtifacts = mode === "single-pass-strict" &&
            String(payload.normalizeConfig.promptVersion ?? "") === "normalizer_v1_1_2" &&
            isSameCaseSet(payload.caseIds, V112_MICRO_CASE_IDS);
        if (shouldWriteV112MicroArtifacts) {
            (0, files_1.ensureDir)(config_1.REPORTS_DIR);
-            (0, files_1.writeJsonFile)(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
-            fs_1.default.writeFileSync(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"), buildMarkdownReport({
+            tryWriteJsonFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.json"), report);
+            tryWriteTextFile(path_1.default.resolve(config_1.REPORTS_DIR, "normalizer_v1_1_2_micro_eval.md"), buildMarkdownReport({
                ...report,
                report_title: "LLM Normalizer v1.1.2 Micro Eval"
-            }), "utf-8");
+            }));
        }
        return report;
    }