Stage 2 завершён: problem-first ответы и follow-up continuity - ассистент переведён от entity-heavy логики к problem-first ответам с problem-unit слоем, удержанием контекста в follow-up и очисткой пользовательского ответа от сырых технических ссылок.

llm_normalizer/backend/dist/services/evalService.js

@@ -9,6 +9,7 @@ const path_1 = __importDefault(require("path"));
 const nanoid_1 = require("nanoid");
 const config_1 = require("../config");
 const stage1Contracts_1 = require("../types/stage1Contracts");
+const stage2EvalContracts_1 = require("../types/stage2EvalContracts");
 const http_1 = require("../utils/http");
 const assistantService_1 = require("./assistantService");
 const assistantSessionStore_1 = require("./assistantSessionStore");
@@ -214,6 +215,20 @@ function isDecisionStateConsistent(decision) {
 const DEFAULT_ASSISTANT_STAGE1_SUITE_FILE = "assistant_stage1_canonical_v0_1.json";
 const ASSISTANT_STAGE1_RUN_SCHEMA_VERSION = "assistant_stage1_eval_run_v0_1";
 const ASSISTANT_STAGE1_COMPARISON_SCHEMA_VERSION = "assistant_stage1_eval_comparison_v0_1";
+const DEFAULT_ASSISTANT_STAGE2_SUITE_FILE = "assistant_stage2_canonical_v0_1.json";
+const ASSISTANT_STAGE2_RUN_SCHEMA_VERSION = "assistant_stage2_eval_run_v0_1";
+const ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION = "assistant_stage2_eval_comparison_v0_1";
+const KNOWN_PROBLEM_UNIT_TYPES = [
+    "document_conflict",
+    "broken_chain_segment",
+    "lifecycle_anomaly_node",
+    "unresolved_settlement_cluster",
+    "period_risk_cluster",
+    "cross_branch_inconsistency_cluster"
+];
+function toProblemUnitType(value) {
+    return KNOWN_PROBLEM_UNIT_TYPES.includes(value) ? value : null;
+}
 function round2(value) {
     return Number(value.toFixed(2));
 }
@@ -258,6 +273,44 @@ function rubricBandForMetric(metric, value) {
     const score = rateToBandScore(metric, value);
     return stage1Contracts_1.ACCOUNTANT_SCORING_RUBRIC_V01[metric].find((item) => item.score === score) ?? null;
 }
+function rateToBandScoreStage2(metric, value) {
+    if (metric === "problem_unit_precision" || metric === "problem_unit_recall_proxy" || metric === "problem_first_answer_rate") {
+        if (value >= 0.75)
+            return 5;
+        if (value >= 0.45)
+            return 3;
+        return 0;
+    }
+    if (metric === "duplicate_collapse_rate") {
+        if (value >= 0.2)
+            return 5;
+        if (value >= 0.08)
+            return 3;
+        return 0;
+    }
+    if (metric === "entity_leakage_rate") {
+        if (value <= 0.2)
+            return 5;
+        if (value <= 0.4)
+            return 3;
+        return 0;
+    }
+    if (metric === "mechanism_coherence_score" || metric === "problem_clarity_score") {
+        if (value >= 4)
+            return 5;
+        if (value >= 2.5)
+            return 3;
+        return 0;
+    }
+    return 0;
+}
+function rubricBandForMetricStage2(metric, value) {
+    if (value === null) {
+        return null;
+    }
+    const score = rateToBandScoreStage2(metric, value);
+    return stage2EvalContracts_1.ASSISTANT_STAGE2_SCORING_RUBRIC_V01[metric].find((item) => item.score === score) ?? null;
+}
 function buildFeatureProfileSnapshot() {
     return {
         FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1: config_1.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1,
@@ -266,7 +319,11 @@ function buildFeatureProfileSnapshot() {
         FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1: process.env.FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1 ?? null,
         FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1: process.env.FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1 ?? null,
         FEATURE_ASSISTANT_INVESTIGATION_STATE_V1: process.env.FEATURE_ASSISTANT_INVESTIGATION_STATE_V1 ?? null,
-        FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1: process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 ?? null
+        FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1: process.env.FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1 ?? null,
+        FEATURE_ASSISTANT_PROBLEM_UNITS_V1: process.env.FEATURE_ASSISTANT_PROBLEM_UNITS_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_UNITS_V1),
+        FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1: process.env.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_CENTRIC_ANSWER_V1),
+        FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1: process.env.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1 ?? String(config_1.FEATURE_ASSISTANT_PROBLEM_UNIT_CONTINUITY_V1),
+        FEATURE_ASSISTANT_STAGE2_EVAL_V1: process.env.FEATURE_ASSISTANT_STAGE2_EVAL_V1 ?? String(config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1)
     };
 }
 function buildCodeVersionMarker() {
@@ -331,6 +388,41 @@ function parseAssistantSuiteFile(inputPath) {
     }
     return parsed;
 }
+function parseAssistantStage2SuiteFile(inputPath) {
+    const filePath = resolveReadablePath(inputPath ?? DEFAULT_ASSISTANT_STAGE2_SUITE_FILE);
+    const raw = fs_1.default.readFileSync(filePath, "utf-8").replace(/^\uFEFF/, "");
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== "object") {
+        throw new Error(`Invalid assistant stage2 suite format: ${filePath}`);
+    }
+    if (!Array.isArray(parsed.cases)) {
+        throw new Error(`Assistant stage2 suite cases[] is required: ${filePath}`);
+    }
+    if (!Array.isArray(parsed.case_ids)) {
+        throw new Error(`Assistant stage2 suite case_ids[] is required: ${filePath}`);
+    }
+    if (typeof parsed.suite_id !== "string" || !parsed.suite_id.trim()) {
+        throw new Error(`Assistant stage2 suite_id is required: ${filePath}`);
+    }
+    if (typeof parsed.suite_version !== "string" || !parsed.suite_version.trim()) {
+        throw new Error(`Assistant stage2 suite_version is required: ${filePath}`);
+    }
+    if (parsed.scenario_count !== parsed.cases.length) {
+        throw new Error(`Assistant stage2 scenario_count mismatch: ${filePath}`);
+    }
+    const declaredIds = [...parsed.case_ids].sort();
+    const actualIds = parsed.cases.map((item) => item.case_id).sort();
+    const idsMatch = declaredIds.length === actualIds.length && declaredIds.every((item, index) => item === actualIds[index]);
+    if (!idsMatch) {
+        throw new Error(`Assistant stage2 case_ids do not match cases[]: ${filePath}`);
+    }
+    for (const item of parsed.cases) {
+        if (!Array.isArray(item.turns) || item.turns.length === 0) {
+            throw new Error(`Assistant stage2 case ${item.case_id} must include at least one turn.`);
+        }
+    }
+    return parsed;
+}
 function hasDomainAnchors(text) {
     const source = String(text ?? "");
     if (!source.trim()) {
@@ -342,6 +434,16 @@ function hasDomainAnchors(text) {
     const hits = [hasPeriod, hasAccountingObject, hasAccountCode].filter(Boolean).length;
     return hits >= 2;
 }
+function detectEntityLeakage(text) {
+    const source = String(text ?? "");
+    if (!source.trim()) {
+        return false;
+    }
+    const uuidHits = source.match(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi)?.length ?? 0;
+    const guidHits = source.match(/\b(?:guid|uuid|entity_id|source_ref|canonical_ref|fragment_id)\b/gi)?.length ?? 0;
+    const longHexHits = source.match(/\b[0-9a-f]{24,}\b/gi)?.length ?? 0;
+    return uuidHits > 0 || guidHits > 1 || longHexHits > 0;
+}
 function extractTextList(value) {
     if (!Array.isArray(value)) {
         return [];
@@ -411,6 +513,50 @@ function buildAssistantEvalMarkdownReport(report) {
         ""
     ].join("\n");
 }
+function buildAssistantStage2EvalMarkdownReport(report) {
+    const metrics = (report.metrics ?? {}).raw ?? {};
+    const bands = (report.rubric_bands ?? {});
+    const subsets = (report.subsets ?? {});
+    const scenarioSummary = (report.scenario_summary ?? {});
+    const rows = Object.keys(metrics)
+        .map((key) => {
+        const rawValue = metrics[key];
+        const band = bands[key];
+        const rawPrintable = rawValue === null || rawValue === undefined ? "n/a" : String(rawValue);
+        const bandPrintable = band ? `${String(band.score)} (${String(band.label)})` : "n/a";
+        return `| ${key} | ${rawPrintable} | ${bandPrintable} |`;
+    })
+        .join("\n");
+    return [
+        `# ${String(report.report_title ?? "Assistant Stage 2 Eval Run")}`,
+        "",
+        `- run_id: ${String(report.run_id ?? "")}`,
+        `- eval_target: ${String(report.eval_target ?? "")}`,
+        `- run_timestamp: ${String(report.run_timestamp ?? "")}`,
+        `- suite_id: ${String(report.suite_id ?? "")}`,
+        `- suite_version: ${String(report.suite_version ?? "")}`,
+        `- cases_total: ${String(report.cases_total ?? 0)}`,
+        "",
+        "## Raw Metrics and Rubric Bands",
+        "",
+        "| Metric | Raw | Rubric band |",
+        "|---|---:|---|",
+        rows || "| n/a | n/a | n/a |",
+        "",
+        "## Subsets",
+        "",
+        `- expected_problem_cases_total: ${String(subsets.expected_problem_cases_total ?? 0)}`,
+        `- followup_cases_total: ${String(subsets.followup_cases_total ?? 0)}`,
+        `- candidate_cases_total: ${String(subsets.candidate_cases_total ?? 0)}`,
+        "",
+        "## Scenario Summary",
+        "",
+        `- improved_or_strong: ${String(scenarioSummary.improved_or_strong ?? 0)}`,
+        `- unchanged_or_mixed: ${String(scenarioSummary.unchanged_or_mixed ?? 0)}`,
+        `- weak_or_regressed: ${String(scenarioSummary.weak_or_regressed ?? 0)}`,
+        ""
+    ].join("\n");
+}
 function buildAssistantComparisonMarkdownReport(report) {
     const metrics = (report.metric_deltas ?? {});
     const summary = (report.scenario_notes_summary ?? {});
@@ -442,6 +588,37 @@ function buildAssistantComparisonMarkdownReport(report) {
         ""
     ].join("\n");
 }
+function buildAssistantStage2ComparisonMarkdownReport(report) {
+    const metrics = (report.metric_deltas ?? {});
+    const summary = (report.scenario_notes_summary ?? {});
+    const rows = Object.keys(metrics)
+        .map((key) => {
+        const row = metrics[key];
+        return `| ${key} | ${String(row.baseline ?? "n/a")} | ${String(row.current ?? "n/a")} | ${String(row.delta ?? "n/a")} | ${String(row.trend ?? "n/a")} |`;
+    })
+        .join("\n");
+    return [
+        `# ${String(report.report_title ?? "Assistant Stage 2 Baseline vs Current")}`,
+        "",
+        `- comparison_id: ${String(report.comparison_id ?? "")}`,
+        `- baseline_run_id: ${String(report.baseline_run_id ?? "")}`,
+        `- current_run_id: ${String(report.current_run_id ?? "")}`,
+        `- suite_version: ${String(report.suite_version ?? "")}`,
+        "",
+        "## Metric Deltas",
+        "",
+        "| Metric | Baseline | Current | Delta | Trend |",
+        "|---|---:|---:|---:|---|",
+        rows || "| n/a | n/a | n/a | n/a | n/a |",
+        "",
+        "## Scenario Notes Summary",
+        "",
+        `- improved: ${String(summary.improved ?? 0)}`,
+        `- unchanged: ${String(summary.unchanged ?? 0)}`,
+        `- weakened: ${String(summary.weakened ?? 0)}`,
+        ""
+    ].join("\n");
+}
 class EvalService {
     normalizerService;
     constructor(normalizerService) {
@@ -806,6 +983,148 @@ class EvalService {
             uncertainty_limitations_count: uncertaintyLimitationsCount
         };
     }
+    collectAssistantStage2Signals(finalResponse, turnResponses) {
+        const base = this.collectAssistantSignals(finalResponse, turnResponses);
+        const debug = finalResponse.debug;
+        const retrievalResults = Array.isArray(debug?.retrieval_results) ? debug.retrieval_results : [];
+        const typeSet = new Set();
+        const mechanismSummaries = new Set();
+        let candidateEvidenceTotal = 0;
+        let problemUnitsTotal = 0;
+        let duplicateCollapsesTotal = 0;
+        for (const result of retrievalResults) {
+            const candidates = Array.isArray(result.candidate_evidence) ? result.candidate_evidence : [];
+            candidateEvidenceTotal += candidates.length;
+            const problemUnits = Array.isArray(result.problem_units) ? result.problem_units : [];
+            problemUnitsTotal += problemUnits.length;
+            for (const unit of problemUnits) {
+                const unitType = toProblemUnitType(unit.problem_unit_type);
+                if (unitType) {
+                    typeSet.add(unitType);
+                }
+                const mechanismSummary = String(unit.mechanism_summary ?? "").trim();
+                if (mechanismSummary) {
+                    mechanismSummaries.add(mechanismSummary);
+                }
+            }
+            if (result.problem_unit_summary && typeof result.problem_unit_summary.duplicate_collapses === "number") {
+                duplicateCollapsesTotal += Number(result.problem_unit_summary.duplicate_collapses);
+            }
+        }
+        const answerMode = typeof debug?.problem_answer_mode === "string" ? debug.problem_answer_mode : null;
+        const unitsUsedCount = Number(debug?.problem_units_used_count ?? 0);
+        const unitIdsUsed = Array.isArray(debug?.problem_unit_ids_used)
+            ? debug.problem_unit_ids_used
+                .map((item) => String(item ?? "").trim())
+                .filter(Boolean)
+            : [];
+        const problemCentricApplied = debug?.problem_centric_answer_applied === true || answerMode === "stage2_problem_centric_v1";
+        return {
+            ...base,
+            candidate_evidence_total: candidateEvidenceTotal,
+            problem_units_total: problemUnitsTotal,
+            problem_unit_types: [...typeSet],
+            problem_mechanism_summaries: [...mechanismSummaries],
+            duplicate_collapses_total: duplicateCollapsesTotal,
+            problem_centric_answer_applied: problemCentricApplied,
+            problem_units_used_count: unitsUsedCount,
+            problem_answer_mode: answerMode,
+            problem_unit_ids_used: unitIdsUsed,
+            entity_leakage_detected: detectEntityLeakage(String(finalResponse.assistant_reply ?? ""))
+        };
+    }
+    getExpectedProblemUnitTypes(suiteCase) {
+        const expected = Array.isArray(suiteCase.expected_hints?.expected_problem_unit_types)
+            ? suiteCase.expected_hints?.expected_problem_unit_types
+            : [];
+        const output = new Set();
+        for (const value of expected ?? []) {
+            const mapped = toProblemUnitType(value);
+            if (mapped) {
+                output.add(mapped);
+            }
+        }
+        return [...output];
+    }
+    computeProblemUnitPrecision(expectedTypes, detectedTypes) {
+        const uniqueExpected = [...new Set(expectedTypes)];
+        const uniqueDetected = [...new Set(detectedTypes)];
+        if (uniqueDetected.length === 0) {
+            return uniqueExpected.length === 0 ? 1 : 0;
+        }
+        if (uniqueExpected.length === 0) {
+            return 0;
+        }
+        const matchedDetected = uniqueDetected.filter((item) => uniqueExpected.includes(item)).length;
+        return round2(matchedDetected / uniqueDetected.length);
+    }
+    computeProblemUnitRecallProxy(expectedTypes, detectedTypes) {
+        const uniqueExpected = [...new Set(expectedTypes)];
+        const uniqueDetected = [...new Set(detectedTypes)];
+        if (uniqueExpected.length === 0) {
+            return null;
+        }
+        if (uniqueDetected.length === 0) {
+            return 0;
+        }
+        const matchedExpected = uniqueExpected.filter((item) => uniqueDetected.includes(item)).length;
+        return round2(matchedExpected / uniqueExpected.length);
+    }
+    computeDuplicateCollapseRate(candidateTotal, duplicateCollapses) {
+        if (candidateTotal <= 0) {
+            return null;
+        }
+        return round2(Math.min(1, Math.max(0, duplicateCollapses / candidateTotal)));
+    }
+    computeMechanismCoherenceScore(finalResponse, signals) {
+        const mechanismBlock = finalResponse.debug?.answer_structure_v11?.mechanism_block;
+        const mechanismStatus = mechanismBlock?.status;
+        const mechanismNotes = extractTextList(mechanismBlock?.mechanism_notes);
+        const hasProblemMechanism = signals.problem_mechanism_summaries.length > 0;
+        let score = 0;
+        if (mechanismStatus === "grounded" && hasProblemMechanism && mechanismNotes.length > 0) {
+            score = 5;
+        }
+        else if ((mechanismStatus === "limited" || mechanismStatus === "unresolved") && (hasProblemMechanism || mechanismNotes.length > 0)) {
+            score = 3;
+        }
+        else if (hasProblemMechanism || mechanismNotes.length > 0) {
+            score = 2;
+        }
+        if (mechanismStatus === "grounded" && !hasProblemMechanism) {
+            score = Math.min(score, 2);
+        }
+        if (signals.limitation_reason_codes.includes("missing_mechanism")) {
+            score -= 1;
+        }
+        return clampScore(score);
+    }
+    computeProblemClarityScore(finalResponse, signals) {
+        const structure = finalResponse.debug?.answer_structure_v11;
+        const answerSummary = String(structure?.answer_summary ?? "").trim();
+        const directAnswer = String(structure?.direct_answer ?? finalResponse.assistant_reply ?? "").trim();
+        const recommendedActions = extractTextList(structure?.next_step_block?.recommended_actions);
+        const clarificationQuestions = extractTextList(structure?.next_step_block?.clarification_questions);
+        const uncertaintyLimitations = extractTextList(structure?.uncertainty_block?.limitations);
+        let score = 0;
+        if (answerSummary.length > 20)
+            score += 1;
+        if (directAnswer.length > 20)
+            score += 1;
+        if (hasDomainAnchors(`${answerSummary} ${directAnswer}`))
+            score += 1;
+        if (recommendedActions.length > 0 || clarificationQuestions.length > 0)
+            score += 1;
+        if (signals.problem_units_total > 0 || signals.problem_centric_answer_applied)
+            score += 1;
+        if ((signals.minimum_evidence_failed || signals.degraded_to === "clarification") && uncertaintyLimitations.length === 0) {
+            score -= 1;
+        }
+        if (signals.entity_leakage_detected) {
+            score -= 1;
+        }
+        return clampScore(score);
+    }
     computeAssistantMetrics(input) {
         const diagnostics = input.diagnostics;
         const total = Math.max(1, diagnostics.length);
@@ -855,6 +1174,68 @@ class EvalService {
             signature_counts: signatureCounter
         };
     }
+    computeAssistantStage2Metrics(input) {
+        const diagnostics = input.diagnostics;
+        const signatureCounter = diagnostics.reduce((acc, item) => {
+            acc[item.signature] = (acc[item.signature] ?? 0) + 1;
+            return acc;
+        }, {});
+        const precisionValues = diagnostics
+            .map((item) => item.problem_unit_precision)
+            .filter((item) => typeof item === "number");
+        const recallValues = diagnostics
+            .map((item) => item.problem_unit_recall_proxy)
+            .filter((item) => typeof item === "number");
+        const collapseValues = diagnostics
+            .map((item) => item.duplicate_collapse_rate)
+            .filter((item) => typeof item === "number");
+        const mechanismValues = diagnostics.map((item) => item.mechanism_coherence_score);
+        const clarityValues = diagnostics.map((item) => item.problem_clarity_score);
+        const firstApplicable = diagnostics.filter((item) => item.problem_first_answer_applied !== null);
+        const firstApplied = firstApplicable.filter((item) => item.problem_first_answer_applied === true).length;
+        const leakageCases = diagnostics.filter((item) => item.entity_leakage).length;
+        const followupCases = diagnostics.filter((item) => item.suite_case.question_type === "followup" || item.turn_count > 1);
+        const candidateCases = diagnostics.filter((item) => item.signals.candidate_evidence_total > 0);
+        const expectedProblemCases = diagnostics.filter((item) => item.expected_problem_first);
+        const average = (values) => {
+            if (values.length === 0)
+                return null;
+            return round2(values.reduce((acc, item) => acc + item, 0) / values.length);
+        };
+        const raw = {
+            problem_unit_precision: average(precisionValues),
+            problem_unit_recall_proxy: average(recallValues),
+            duplicate_collapse_rate: average(collapseValues),
+            mechanism_coherence_score: average(mechanismValues),
+            problem_clarity_score: average(clarityValues),
+            problem_first_answer_rate: firstApplicable.length > 0 ? round2(firstApplied / firstApplicable.length) : null,
+            entity_leakage_rate: diagnostics.length > 0 ? round2(leakageCases / diagnostics.length) : null
+        };
+        const rubric_bands = {
+            problem_unit_precision: rubricBandForMetricStage2("problem_unit_precision", raw.problem_unit_precision),
+            problem_unit_recall_proxy: rubricBandForMetricStage2("problem_unit_recall_proxy", raw.problem_unit_recall_proxy),
+            duplicate_collapse_rate: rubricBandForMetricStage2("duplicate_collapse_rate", raw.duplicate_collapse_rate),
+            mechanism_coherence_score: rubricBandForMetricStage2("mechanism_coherence_score", raw.mechanism_coherence_score),
+            problem_clarity_score: rubricBandForMetricStage2("problem_clarity_score", raw.problem_clarity_score),
+            problem_first_answer_rate: rubricBandForMetricStage2("problem_first_answer_rate", raw.problem_first_answer_rate),
+            entity_leakage_rate: rubricBandForMetricStage2("entity_leakage_rate", raw.entity_leakage_rate)
+        };
+        return {
+            raw,
+            rubric_bands,
+            denominators: {
+                cases_total: diagnostics.length,
+                expected_problem_cases_total: expectedProblemCases.length,
+                followup_cases_total: followupCases.length,
+                candidate_cases_total: candidateCases.length,
+                precision_cases_total: precisionValues.length,
+                recall_cases_total: recallValues.length,
+                duplicate_collapse_cases_total: collapseValues.length,
+                problem_first_applicable_cases_total: firstApplicable.length
+            },
+            signature_counts: signatureCounter
+        };
+    }
     buildAssistantComparisonReport(input) {
         const baselinePath = resolveReadablePath(input.baselineReportFile);
         const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
@@ -959,6 +1340,122 @@ class EvalService {
             }
         };
     }
+    buildAssistantStage2ComparisonReport(input) {
+        const baselinePath = resolveReadablePath(input.baselineReportFile);
+        const baselineReport = JSON.parse(fs_1.default.readFileSync(baselinePath, "utf-8"));
+        const currentReport = input.currentReport;
+        const metricKeys = [
+            "problem_unit_precision",
+            "problem_unit_recall_proxy",
+            "duplicate_collapse_rate",
+            "mechanism_coherence_score",
+            "problem_clarity_score",
+            "problem_first_answer_rate",
+            "entity_leakage_rate"
+        ];
+        const lowerIsBetter = new Set(["entity_leakage_rate"]);
+        const baselineRaw = (baselineReport.metrics ?? {}).raw ?? {};
+        const currentRaw = (currentReport.metrics ?? {}).raw ?? {};
+        const deltas = {};
+        for (const metric of metricKeys) {
+            const baseline = typeof baselineRaw[metric] === "number" ? Number(baselineRaw[metric]) : null;
+            const current = typeof currentRaw[metric] === "number" ? Number(currentRaw[metric]) : null;
+            const delta = baseline !== null && current !== null ? round2(current - baseline) : null;
+            let trend = "n/a";
+            if (baseline !== null && current !== null) {
+                const improved = lowerIsBetter.has(metric) ? current < baseline - 0.01 : current > baseline + 0.01;
+                const weakened = lowerIsBetter.has(metric) ? current > baseline + 0.01 : current < baseline - 0.01;
+                trend = improved ? "improved" : weakened ? "weakened" : "unchanged";
+            }
+            deltas[metric] = { baseline, current, delta, trend };
+        }
+        const baselineResults = Array.isArray(baselineReport.results) ? baselineReport.results : [];
+        const currentResults = Array.isArray(currentReport.results) ? currentReport.results : [];
+        const baselineByCase = new Map();
+        for (const row of baselineResults) {
+            baselineByCase.set(String(row.case_id ?? ""), row);
+        }
+        const improvedNotes = [];
+        const unchangedNotes = [];
+        const weakenedNotes = [];
+        const toComposite = (row) => {
+            if (!row || typeof row !== "object")
+                return null;
+            const metricSubscores = row.metric_subscores;
+            if (!metricSubscores)
+                return null;
+            const clarity = typeof metricSubscores.problem_clarity_score === "number" ? Number(metricSubscores.problem_clarity_score) : null;
+            const mechanism = typeof metricSubscores.mechanism_coherence_score === "number" ? Number(metricSubscores.mechanism_coherence_score) : null;
+            const firstRate = typeof metricSubscores.problem_first_answer_rate === "number" ? Number(metricSubscores.problem_first_answer_rate) : null;
+            const leakageRate = typeof metricSubscores.entity_leakage_rate === "number" ? Number(metricSubscores.entity_leakage_rate) : null;
+            if (clarity === null || mechanism === null || firstRate === null || leakageRate === null) {
+                return null;
+            }
+            return round2((clarity + mechanism + firstRate * 5 + (1 - leakageRate) * 5) / 4);
+        };
+        for (const row of currentResults) {
+            const caseId = String(row.case_id ?? "");
+            const currentComposite = toComposite(row);
+            const baselineComposite = toComposite(baselineByCase.get(caseId));
+            if (currentComposite === null || baselineComposite === null) {
+                continue;
+            }
+            const delta = round2(currentComposite - baselineComposite);
+            const note = `${caseId}: composite ${baselineComposite} -> ${currentComposite} (delta ${delta})`;
+            if (delta > 0.25) {
+                improvedNotes.push(note);
+            }
+            else if (delta < -0.25) {
+                weakenedNotes.push(note);
+            }
+            else {
+                unchangedNotes.push(note);
+            }
+        }
+        const comparisonId = `assistant-stage2-compare-${(0, nanoid_1.nanoid)(8)}`;
+        const comparisonReport = {
+            schema_version: ASSISTANT_STAGE2_COMPARISON_SCHEMA_VERSION,
+            comparison_id: comparisonId,
+            run_timestamp: new Date().toISOString(),
+            baseline_run_id: baselineReport.run_id ?? null,
+            current_run_id: currentReport.run_id ?? null,
+            eval_target: "assistant_stage2",
+            suite_id: currentReport.suite_id ?? baselineReport.suite_id ?? null,
+            suite_version: currentReport.suite_version ?? baselineReport.suite_version ?? null,
+            baseline_report_file: baselinePath,
+            current_report_file: currentReport.artifacts && typeof currentReport.artifacts === "object"
+                ? currentReport.artifacts.run_report_json_path ?? null
+                : null,
+            metric_deltas: deltas,
+            scenario_notes_summary: {
+                improved: improvedNotes.length,
+                unchanged: unchangedNotes.length,
+                weakened: weakenedNotes.length
+            },
+            scenario_notes: {
+                improved: improvedNotes,
+                unchanged: unchangedNotes,
+                weakened: weakenedNotes
+            },
+            known_limitations: currentReport.known_limitations ?? [
+                "Stage 2 comparison remains run-to-run and depends on stable feature profile.",
+                "Metrics are Stage 2 Wave 5 heuristics, not final product scorecards."
+            ],
+            report_title: "Assistant Stage 2 Baseline vs Current"
+        };
+        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
+        const jsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.json`);
+        const mdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${comparisonId}.md`);
+        (0, files_1.writeJsonFile)(jsonPath, comparisonReport);
+        fs_1.default.writeFileSync(mdPath, buildAssistantStage2ComparisonMarkdownReport(comparisonReport), "utf-8");
+        return {
+            ...comparisonReport,
+            artifacts: {
+                comparison_report_json_path: jsonPath,
+                comparison_report_md_path: mdPath
+            }
+        };
+    }
     async runAssistantStage1(payload) {
         if (!config_1.FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1) {
             throw new http_1.ApiError("ASSISTANT_STAGE1_EVAL_DISABLED", "Assistant Stage 1 eval target is disabled by FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1.", 409);
@@ -1290,6 +1787,278 @@ class EvalService {
         }
         return report;
     }
+    async runAssistantStage2(payload) {
+        if (!config_1.FEATURE_ASSISTANT_STAGE2_EVAL_V1) {
+            throw new http_1.ApiError("ASSISTANT_STAGE2_EVAL_DISABLED", "Assistant Stage 2 eval target is disabled by FEATURE_ASSISTANT_STAGE2_EVAL_V1.", 409);
+        }
+        const suite = parseAssistantStage2SuiteFile(payload.caseSetFile);
+        const suiteCases = suite.cases.filter((item) => !payload.caseIds || payload.caseIds.includes(item.case_id));
+        const runId = `assistant-stage2-${(0, nanoid_1.nanoid)(10)}`;
+        const assistantService = new assistantService_1.AssistantService(this.normalizerService, new assistantSessionStore_1.AssistantSessionStore());
+        const diagnostics = [];
+        let requestsTotal = 0;
+        for (const suiteCase of suiteCases) {
+            const sessionId = `${runId}-${suiteCase.case_id}`;
+            const turnResponses = [];
+            const notes = [];
+            const limitations = [];
+            const expectedProblemUnitTypes = this.getExpectedProblemUnitTypes(suiteCase);
+            const expectedProblemFirst = suiteCase.expected_hints?.expected_problem_first ?? (suiteCase.broadness_level !== "low" || suiteCase.question_type !== "direct");
+            try {
+                for (const turn of suiteCase.turns) {
+                    const response = await assistantService.handleMessage({
+                        session_id: sessionId,
+                        user_message: turn.user_message,
+                        message: turn.user_message,
+                        mode: "assistant",
+                        apiKey: payload.normalizeConfig.apiKey,
+                        model: payload.normalizeConfig.model,
+                        baseUrl: payload.normalizeConfig.baseUrl,
+                        temperature: payload.normalizeConfig.temperature,
+                        maxOutputTokens: payload.normalizeConfig.maxOutputTokens,
+                        promptVersion: payload.normalizeConfig.promptVersion,
+                        systemPrompt: payload.normalizeConfig.systemPrompt,
+                        developerPrompt: payload.normalizeConfig.developerPrompt,
+                        domainPrompt: payload.normalizeConfig.domainPrompt,
+                        fewShotExamples: payload.normalizeConfig.fewShotExamples,
+                        useMock: payload.useMock
+                    });
+                    turnResponses.push(response);
+                    requestsTotal += 1;
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                diagnostics.push({
+                    suite_case: suiteCase,
+                    session_id: sessionId,
+                    trace_id: null,
+                    final_reply_type: "backend_error",
+                    turn_count: turnResponses.length,
+                    signature: `backend_error|${suiteCase.scenario_tag}`,
+                    expected_problem_unit_types: expectedProblemUnitTypes,
+                    expected_problem_first: expectedProblemFirst,
+                    problem_unit_precision: 0,
+                    problem_unit_recall_proxy: expectedProblemUnitTypes.length > 0 ? 0 : null,
+                    duplicate_collapse_rate: null,
+                    mechanism_coherence_score: 0,
+                    problem_clarity_score: 0,
+                    problem_first_answer_applied: expectedProblemFirst ? false : null,
+                    entity_leakage: false,
+                    signals: {
+                        broad_query_detected: suiteCase.broadness_level !== "low",
+                        broad_result_flag: false,
+                        narrowing_strength: null,
+                        minimum_evidence_failed: true,
+                        degraded_to: "clarification",
+                        evidence_confidence: "low",
+                        limitation_reason_codes: [],
+                        mechanism_status: null,
+                        source_refs: [],
+                        routes: [],
+                        followup_state_applied: false,
+                        uncertainty_limitations_count: 0,
+                        candidate_evidence_total: 0,
+                        problem_units_total: 0,
+                        problem_unit_types: [],
+                        problem_mechanism_summaries: [],
+                        duplicate_collapses_total: 0,
+                        problem_centric_answer_applied: false,
+                        problem_units_used_count: 0,
+                        problem_answer_mode: null,
+                        problem_unit_ids_used: [],
+                        entity_leakage_detected: false
+                    },
+                    limitations: [errorMessage],
+                    notes: [`Case execution failed: ${errorMessage}`]
+                });
+                continue;
+            }
+            const finalResponse = turnResponses[turnResponses.length - 1];
+            const signals = this.collectAssistantStage2Signals(finalResponse, turnResponses);
+            const problemUnitPrecision = this.computeProblemUnitPrecision(expectedProblemUnitTypes, signals.problem_unit_types);
+            const problemUnitRecallProxy = this.computeProblemUnitRecallProxy(expectedProblemUnitTypes, signals.problem_unit_types);
+            const duplicateCollapseRate = this.computeDuplicateCollapseRate(signals.candidate_evidence_total, signals.duplicate_collapses_total);
+            const mechanismCoherenceScore = this.computeMechanismCoherenceScore(finalResponse, signals);
+            const problemClarityScore = this.computeProblemClarityScore(finalResponse, signals);
+            const problemFirstAnswerApplied = expectedProblemFirst ? signals.problem_centric_answer_applied && signals.problem_units_used_count > 0 : null;
+            if (signals.problem_units_total === 0 && expectedProblemUnitTypes.length > 0) {
+                limitations.push("missing_problem_units");
+            }
+            if (signals.problem_centric_answer_applied && signals.problem_units_used_count <= 0) {
+                limitations.push("problem_mode_without_units");
+            }
+            limitations.push(...signals.limitation_reason_codes.map((item) => `limitation_reason:${item}`));
+            if (signals.entity_leakage_detected) {
+                limitations.push("entity_leakage_detected");
+            }
+            if (problemFirstAnswerApplied === false)
+                notes.push("problem_first_not_applied");
+            if (signals.problem_units_total === 0)
+                notes.push("problem_units_missing");
+            if (signals.problem_unit_types.length > 0)
+                notes.push(`problem_types:${signals.problem_unit_types.join(",")}`);
+            if (signals.entity_leakage_detected)
+                notes.push("entity_leakage");
+            if (signals.degraded_to === "clarification")
+                notes.push("clarification_degraded");
+            diagnostics.push({
+                suite_case: suiteCase,
+                session_id: sessionId,
+                trace_id: finalResponse.debug?.trace_id ?? null,
+                final_reply_type: finalResponse.reply_type,
+                turn_count: suiteCase.turns.length,
+                signature: [
+                    finalResponse.reply_type,
+                    signals.problem_answer_mode ?? "unknown",
+                    signals.problem_unit_types.sort().join(","),
+                    signals.degraded_to ?? "none"
+                ].join("|"),
+                expected_problem_unit_types: expectedProblemUnitTypes,
+                expected_problem_first: expectedProblemFirst,
+                problem_unit_precision: problemUnitPrecision,
+                problem_unit_recall_proxy: problemUnitRecallProxy,
+                duplicate_collapse_rate: duplicateCollapseRate,
+                mechanism_coherence_score: mechanismCoherenceScore,
+                problem_clarity_score: problemClarityScore,
+                problem_first_answer_applied: problemFirstAnswerApplied,
+                entity_leakage: signals.entity_leakage_detected,
+                signals,
+                limitations: Array.from(new Set(limitations)),
+                notes
+            });
+        }
+        const metrics = this.computeAssistantStage2Metrics({ diagnostics });
+        const caseRecords = diagnostics.map((item) => {
+            const caseMetricVector = {
+                problem_unit_precision: item.problem_unit_precision,
+                problem_unit_recall_proxy: item.problem_unit_recall_proxy,
+                duplicate_collapse_rate: item.duplicate_collapse_rate,
+                mechanism_coherence_score: round2(item.mechanism_coherence_score),
+                problem_clarity_score: round2(item.problem_clarity_score),
+                problem_first_answer_rate: item.problem_first_answer_applied === null ? null : item.problem_first_answer_applied ? 1 : 0,
+                entity_leakage_rate: item.entity_leakage ? 1 : 0
+            };
+            return {
+                schema_version: stage2EvalContracts_1.ASSISTANT_STAGE2_EVAL_RECORD_SCHEMA_VERSION,
+                created_at: new Date().toISOString(),
+                case_id: item.suite_case.case_id,
+                scenario_tag: item.suite_case.scenario_tag,
+                session_id: item.session_id,
+                trace_id: item.trace_id,
+                question_type: item.suite_case.question_type,
+                broadness_level: item.suite_case.broadness_level,
+                expected_problem_unit_types: item.expected_problem_unit_types,
+                expected_problem_first: item.expected_problem_first,
+                problem_units_detected: item.signals.problem_units_total,
+                candidate_evidence_detected: item.signals.candidate_evidence_total,
+                duplicate_collapses_detected: item.signals.duplicate_collapses_total,
+                metric_subscores: caseMetricVector,
+                raw_signals: {
+                    final_reply_type: item.final_reply_type,
+                    turn_count: item.turn_count,
+                    broad_query_detected: item.signals.broad_query_detected,
+                    broad_result_flag: item.signals.broad_result_flag,
+                    narrowing_strength: item.signals.narrowing_strength,
+                    minimum_evidence_failed: item.signals.minimum_evidence_failed,
+                    degraded_to: item.signals.degraded_to,
+                    evidence_confidence: item.signals.evidence_confidence,
+                    limitation_reason_codes: item.signals.limitation_reason_codes,
+                    mechanism_status: item.signals.mechanism_status,
+                    source_refs: item.signals.source_refs,
+                    routes: item.signals.routes,
+                    followup_state_applied: item.signals.followup_state_applied,
+                    problem_units_total: item.signals.problem_units_total,
+                    candidate_evidence_total: item.signals.candidate_evidence_total,
+                    problem_unit_types: item.signals.problem_unit_types,
+                    duplicate_collapses_total: item.signals.duplicate_collapses_total,
+                    problem_centric_answer_applied: item.signals.problem_centric_answer_applied,
+                    problem_units_used_count: item.signals.problem_units_used_count,
+                    problem_answer_mode: item.signals.problem_answer_mode,
+                    problem_unit_ids_used: item.signals.problem_unit_ids_used,
+                    entity_leakage_detected: item.signals.entity_leakage_detected
+                },
+                limitations: item.limitations,
+                notes: item.notes
+            };
+        });
+        const strongestSignals = Object.entries(metrics.rubric_bands)
+            .filter(([, band]) => band?.score === 5)
+            .map(([name]) => name);
+        const weakestSignals = Object.entries(metrics.rubric_bands)
+            .filter(([, band]) => band?.score === 0)
+            .map(([name]) => name);
+        const runTimestamp = new Date().toISOString();
+        const report = {
+            schema_version: ASSISTANT_STAGE2_RUN_SCHEMA_VERSION,
+            run_id: runId,
+            run_timestamp: runTimestamp,
+            eval_target: "assistant_stage2",
+            mode: payload.mode,
+            use_mock: Boolean(payload.useMock),
+            prompt_version: payload.normalizeConfig.promptVersion ?? null,
+            suite_id: suite.suite_id,
+            suite_version: suite.suite_version,
+            suite_schema_version: suite.schema_version ?? null,
+            scenario_count: suite.scenario_count,
+            case_ids: suiteCases.map((item) => item.case_id),
+            cases_total: caseRecords.length,
+            feature_profile_snapshot: buildFeatureProfileSnapshot(),
+            code_version: buildCodeVersionMarker(),
+            metrics: {
+                raw: metrics.raw,
+                denominators: metrics.denominators
+            },
+            rubric_bands: metrics.rubric_bands,
+            subsets: {
+                expected_problem_cases_total: metrics.denominators.expected_problem_cases_total,
+                followup_cases_total: metrics.denominators.followup_cases_total,
+                candidate_cases_total: metrics.denominators.candidate_cases_total
+            },
+            budget: {
+                requests_total: requestsTotal
+            },
+            results: caseRecords,
+            scenario_summary: {
+                improved_or_strong: caseRecords.filter((item) => {
+                    const clarity = Number(item.metric_subscores.problem_clarity_score ?? 0);
+                    const mechanism = Number(item.metric_subscores.mechanism_coherence_score ?? 0);
+                    return clarity >= 4 && mechanism >= 3;
+                }).length,
+                unchanged_or_mixed: caseRecords.filter((item) => {
+                    const clarity = Number(item.metric_subscores.problem_clarity_score ?? 0);
+                    return clarity >= 2.5 && clarity < 4;
+                }).length,
+                weak_or_regressed: caseRecords.filter((item) => Number(item.metric_subscores.problem_clarity_score ?? 0) < 2.5).length
+            },
+            improvement_hints: {
+                strongest_signals: strongestSignals.length > 0 ? strongestSignals.join(", ") : "none",
+                weakest_signals: weakestSignals.length > 0 ? weakestSignals.join(", ") : "none"
+            },
+            known_limitations: [
+                "Stage 2 eval remains heuristic and scoped to problem-unit baseline (no graph/lifecycle/investigation runtime scoring).",
+                "problem_unit_recall_proxy uses suite expected types as lightweight proxy, not full ground-truth labeling.",
+                "Comparison quality depends on stable feature profile and reproducible mock/runtime setup."
+            ],
+            report_title: "Assistant Stage 2 Eval Run"
+        };
+        (0, files_1.ensureDir)(config_1.REPORTS_DIR);
+        const runJsonPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.json`);
+        const runMdPath = path_1.default.resolve(config_1.REPORTS_DIR, `${runId}.md`);
+        (0, files_1.writeJsonFile)(runJsonPath, report);
+        fs_1.default.writeFileSync(runMdPath, buildAssistantStage2EvalMarkdownReport(report), "utf-8");
+        report.artifacts = {
+            run_report_json_path: runJsonPath,
+            run_report_md_path: runMdPath
+        };
+        if (payload.compareWithReportFile) {
+            report.comparison = this.buildAssistantStage2ComparisonReport({
+                currentReport: report,
+                baselineReportFile: payload.compareWithReportFile
+            });
+        }
+        return report;
+    }
     async run(payload) {
         const mode = payload.mode ?? "standard";
         const evalTarget = payload.evalTarget ?? "normalizer";
@@ -1303,6 +2072,16 @@ class EvalService {
                 compareWithReportFile: payload.compareWithReportFile
             });
         }
+        if (evalTarget === "assistant_stage2") {
+            return this.runAssistantStage2({
+                normalizeConfig: payload.normalizeConfig,
+                caseIds: payload.caseIds,
+                useMock: payload.useMock,
+                mode,
+                caseSetFile: payload.caseSetFile,
+                compareWithReportFile: payload.compareWithReportFile
+            });
+        }
         const promptVersion = String(payload.normalizeConfig.promptVersion ?? "").toLowerCase();
         const schemaVersion = String(payload.normalizeConfig.schemaVersion ?? "").toLowerCase();
         const isV2 = promptVersion.startsWith("normalizer_v2") || schemaVersion === "v2" || schemaVersion === "v2_0_1" || schemaVersion === "v2_0_2";