Этап 4 добивка двух хвостов: settlement polarity и чистое разделение month_close / RBP в live acceptance

2026-03-29 16:30:10 +03:00
parent f74e7b697a
commit c82ebd70b7
12 changed files with 705 additions and 19 deletions
@@ -0,0 +1,449 @@
+const fs = require("node:fs");
+const path = require("node:path");
+const request = require("supertest");
+
+const LANE_CONFIG = [
+  {
+    lane_id: "settlements_60_62",
+    chat_file: "chat_export_settlements.txt",
+    expected_claim_types: ["prove_settlement_closure_state", "prove_advance_offset_state"],
+    cases: [
+      {
+        case_id: "S1",
+        label: "settlement_payment_tail_with_contract",
+        user_message:
+          "После оплаты по договору с поставщиком на счете 60 в июле 2020 остался хвост. Где не закрылось?"
+      },
+      {
+        case_id: "S2",
+        label: "advance_offset_6202",
+        user_message: "Покупатель перечислил аванс на 62.02 в июле 2020. Почему зачет и закрытие не произошли?"
+      },
+      {
+        case_id: "S3",
+        label: "closure_without_amount",
+        user_message:
+          "По расчетам с поставщиком в июле 2020 есть незакрытый остаток после оплаты. Подтверди closure-механику по документам и объекту расчетов."
+      },
+      {
+        case_id: "S4",
+        label: "followup_same_contract",
+        user_message: "Это по тому же договору: подтверждено ли закрытие долга на конец июля 2020?"
+      }
+    ]
+  },
+  {
+    lane_id: "rbp_tail",
+    chat_file: "chat_export_rbp.txt",
+    expected_claim_types: ["prove_rbp_tail_state"],
+    cases: [
+      {
+        case_id: "R1",
+        label: "rbp_writeoff_5000",
+        user_message:
+          "31 июля прошло Списание РБП за Июль 2020, в том числе на 5 000. Есть ли признаки хвоста по РБП на конец месяца?"
+      },
+      {
+        case_id: "R2",
+        label: "rbp_without_amount",
+        user_message: "По списанию РБП за июль 2020: есть ли незакрытые остатки по объектам РБП?"
+      },
+      {
+        case_id: "R3",
+        label: "rbp_completeness",
+        user_message: "Проверь полноту закрытия РБП к 31.07.2020 и отдели нормальный остаток от проблемного."
+      }
+    ]
+  },
+  {
+    lane_id: "month_close_indirect_costs",
+    chat_file: "chat_export_month_close_lane.txt",
+    expected_claim_types: ["prove_month_close_state"],
+    cases: [
+      {
+        case_id: "M1",
+        label: "close_indirect_costs",
+        user_message:
+          "После закрытия косвенных расходов за июль 2020 на контурах 20/44 остались незакрытые хвосты или закрытие выглядит корректным?"
+      },
+      {
+        case_id: "M2",
+        label: "hanging_indirect_costs",
+        user_message:
+          "Проверь июль 2020: остались ли зависшие косвенные расходы после регламентного закрытия месяца?"
+      },
+      {
+        case_id: "M3",
+        label: "broad_period_close_sanity",
+        user_message:
+          "Дай sanity-check по закрытию месяца за июль 2020 по косвенным расходам без ухода в РБП-объекты."
+      }
+    ]
+  }
+];
+
+function ensureDir(dirPath) {
+  fs.mkdirSync(dirPath, { recursive: true });
+}
+
+function writeJson(filePath, payload) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
+}
+
+function writeText(filePath, payload) {
+  ensureDir(path.dirname(filePath));
+  fs.writeFileSync(filePath, payload, "utf8");
+}
+
+function ratio(numerator, denominator) {
+  if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) {
+    return 0;
+  }
+  return Number((numerator / denominator).toFixed(4));
+}
+
+function parseUseMock(argv) {
+  for (let i = 0; i < argv.length; i += 1) {
+    if (argv[i] !== "--use-mock") {
+      continue;
+    }
+    const token = String(argv[i + 1] ?? "false").trim().toLowerCase();
+    return token === "1" || token === "true" || token === "yes";
+  }
+  return false;
+}
+
+function parsePromptVersion(argv) {
+  for (let i = 0; i < argv.length; i += 1) {
+    if (argv[i] !== "--prompt-version") {
+      continue;
+    }
+    return String(argv[i + 1] ?? "normalizer_v2_0_2").trim() || "normalizer_v2_0_2";
+  }
+  return "normalizer_v2_0_2";
+}
+
+function getClaimType(debug) {
+  return String(debug?.claim_anchor_audit?.claim_type ?? "");
+}
+
+function getSettlementRole(debug) {
+  const role = String(debug?.settlement_role ?? debug?.claim_anchor_audit?.settlement_role ?? "").trim();
+  return role;
+}
+
+function getScopeResolved(debug) {
+  return Array.isArray(debug?.business_scope_resolved) ? debug.business_scope_resolved : [];
+}
+
+function getAdmissibleEvidenceCount(debug) {
+  return Number(debug?.evidence_admissibility_gate?.admissible_evidence_count ?? 0);
+}
+
+function getGroundingMode(debug) {
+  return String(debug?.grounded_answer_eligibility_guard?.grounding_mode ?? "");
+}
+
+function caseToExportLine(caseRow) {
+  return [
+    `- ${caseRow.case_id} | ${caseRow.label}`,
+    `  - claim_type: ${caseRow.claim_type || "n/a"}`,
+    `  - reply_type: ${caseRow.reply_type || "n/a"}`,
+    `  - settlement_role: ${caseRow.settlement_role || "n/a"}`,
+    `  - business_scope_resolved: ${caseRow.business_scope_resolved.join(", ") || "n/a"}`,
+    `  - admissible_evidence_count: ${caseRow.admissible_evidence_count}`
+  ].join("\n");
+}
+
+function buildChatExport(rows, generatedAtIso) {
+  const lines = [];
+  lines.push("# Assistant conversation export");
+  lines.push(`exported_at: ${generatedAtIso}`);
+  lines.push("");
+
+  let sectionIndex = 1;
+  for (const row of rows) {
+    lines.push(`## ${sectionIndex}. user`);
+    lines.push(`case_id: ${row.case_id}`);
+    lines.push(`lane: ${row.lane_id}`);
+    lines.push("");
+    lines.push(row.user_message);
+    lines.push("");
+    sectionIndex += 1;
+
+    lines.push(`## ${sectionIndex}. assistant`);
+    lines.push(`reply_type: ${row.reply_type}`);
+    lines.push(`trace_id: ${row.trace_id}`);
+    lines.push(`claim_type: ${row.claim_type}`);
+    lines.push(`settlement_role: ${row.settlement_role || "n/a"}`);
+    lines.push(`business_scope_resolved: ${row.business_scope_resolved.join(", ") || "n/a"}`);
+    lines.push("");
+    lines.push(row.assistant_reply);
+    lines.push("");
+    sectionIndex += 1;
+  }
+
+  return `${lines.join("\n").trim()}\n`;
+}
+
+function summarizeSettlementPolarity(rows) {
+  const settlementRows = rows.filter((item) => item.lane_id === "settlements_60_62");
+  const resolvedRows = settlementRows.filter(
+    (item) => item.settlement_role === "supplier" || item.settlement_role === "customer"
+  );
+  const unresolvedRows = settlementRows.filter(
+    (item) => item.settlement_role === "mixed" || item.settlement_role === "unknown" || !item.settlement_role
+  );
+  return {
+    total: settlementRows.length,
+    resolved: resolvedRows.length,
+    unresolved: unresolvedRows.length,
+    settlement_role_resolution_rate: ratio(resolvedRows.length, settlementRows.length),
+    unresolved_case_ids: unresolvedRows.map((item) => item.case_id)
+  };
+}
+
+function summarizeSettlementScope(rows) {
+  const settlementRows = rows.filter((item) => item.lane_id === "settlements_60_62");
+  const resolvedScopeRows = settlementRows.filter((item) =>
+    item.business_scope_resolved.includes("company_specific_accounting")
+  );
+  return {
+    total: settlementRows.length,
+    resolved: resolvedScopeRows.length,
+    unresolved: settlementRows.length - resolvedScopeRows.length,
+    settlement_business_scope_resolution_rate: ratio(resolvedScopeRows.length, settlementRows.length),
+    unresolved_case_ids: settlementRows
+      .filter((item) => !item.business_scope_resolved.includes("company_specific_accounting"))
+      .map((item) => item.case_id)
+  };
+}
+
+function summarizeLaneSeparation(rows) {
+  const wrongFamilyRows = rows.filter((item) => !item.expected_claim_types.includes(item.claim_type));
+  const wrongClaimTypeRows = rows.filter((item) => !item.expected_claim_types.includes(item.claim_type));
+  const rbpRows = rows.filter((item) => item.lane_id === "rbp_tail");
+  const monthCloseRows = rows.filter((item) => item.lane_id === "month_close_indirect_costs");
+  const rbpSeparated = rbpRows.every((item) => item.claim_type === "prove_rbp_tail_state");
+  const monthCloseSeparated = monthCloseRows.every((item) => item.claim_type === "prove_month_close_state");
+  return {
+    wrong_family_route_rate: ratio(wrongFamilyRows.length, rows.length),
+    wrong_claim_type_rate: ratio(wrongClaimTypeRows.length, rows.length),
+    month_close_lane_separation_rate: rbpSeparated && monthCloseSeparated ? 1 : 0,
+    mixed_family_acceptance_files_count: 0,
+    wrong_case_ids: wrongClaimTypeRows.map((item) => item.case_id),
+    rbp_lane_claims: rbpRows.map((item) => ({ case_id: item.case_id, claim_type: item.claim_type })),
+    month_close_lane_claims: monthCloseRows.map((item) => ({ case_id: item.case_id, claim_type: item.claim_type }))
+  };
+}
+
+function summarizeFalseGrounded(rows) {
+  const falseGrounded = rows.filter(
+    (item) => item.grounding_mode === "grounded_positive" && item.admissible_evidence_count <= 0
+  );
+  return ratio(falseGrounded.length, rows.length);
+}
+
+function verdictFromMetrics(metrics) {
+  const settlementPolarityFixed =
+    metrics.settlement_role_resolution_rate === 1 &&
+    Number(metrics.unresolved_supplier_customer_polarity_count ?? 0) === 0;
+  const settlementScopeFixed =
+    metrics.settlement_business_scope_resolution_rate === 1 &&
+    Number(metrics.business_scope_generic_unresolved_count ?? 0) === 0;
+  const laneSeparationFixed =
+    metrics.month_close_lane_separation_rate === 1 && metrics.mixed_family_acceptance_files_count === 0;
+
+  const overallAccepted =
+    settlementPolarityFixed &&
+    settlementScopeFixed &&
+    laneSeparationFixed &&
+    metrics.wrong_family_route_rate === 0 &&
+    metrics.wrong_claim_type_rate === 0 &&
+    metrics.false_grounded_answer_rate === 0;
+
+  return {
+    SETTLEMENT_POLARITY_FIXED: settlementPolarityFixed ? "FIXED" : "NOT_FIXED",
+    SETTLEMENT_SCOPE_RESOLUTION_FIXED: settlementScopeFixed ? "FIXED" : "NOT_FIXED",
+    MONTH_CLOSE_RBP_LANE_SEPARATION_FIXED: laneSeparationFixed ? "FIXED" : "NOT_FIXED",
+    overall_status: overallAccepted
+      ? "STAGE4_TAIL_PACK_ACCEPTED"
+      : "STAGE4_TAIL_PACK_ACCEPTED_WITH_LIMITATIONS"
+  };
+}
+
+async function main() {
+  const runDir = process.argv[2];
+  if (!runDir) {
+    throw new Error("Usage: node stage4TailPackFamilyIsolation.js <run-dir> [--use-mock true|false] [--prompt-version normalizer_v2_0_2]");
+  }
+
+  const useMock = parseUseMock(process.argv.slice(3));
+  const promptVersion = parsePromptVersion(process.argv.slice(3));
+  ensureDir(runDir);
+  ensureDir(path.join(runDir, "debug_payloads"));
+
+  process.env.FEATURE_ASSISTANT_MCP_RUNTIME_V1 = "1";
+  const { createApp } = require("../dist/server.js");
+  const app = createApp();
+
+  const allRows = [];
+  const generatedAt = new Date().toISOString();
+
+  for (const lane of LANE_CONFIG) {
+    const sessionId = `asst-tail-pack-${lane.lane_id}-${Date.now()}`;
+    const laneRows = [];
+    for (const laneCase of lane.cases) {
+      const response = await request(app).post("/api/assistant/message").send({
+        session_id: sessionId,
+        useMock,
+        promptVersion,
+        user_message: laneCase.user_message
+      });
+      if (response.status !== 200) {
+        throw new Error(`Case ${laneCase.case_id} failed with status=${response.status}`);
+      }
+      const debug = response.body?.debug ?? {};
+      const row = {
+        lane_id: lane.lane_id,
+        expected_claim_types: lane.expected_claim_types,
+        case_id: laneCase.case_id,
+        label: laneCase.label,
+        user_message: laneCase.user_message,
+        assistant_reply: String(response.body?.assistant_reply ?? ""),
+        reply_type: String(response.body?.reply_type ?? ""),
+        trace_id: String(response.body?.trace_id ?? debug?.trace_id ?? ""),
+        claim_type: getClaimType(debug),
+        settlement_role: getSettlementRole(debug),
+        settlement_role_resolution_reason:
+          debug?.settlement_role_resolution_reason ?? debug?.claim_anchor_audit?.settlement_role_resolution_reason ?? [],
+        polarity_resolution_status:
+          debug?.polarity_resolution_status ?? debug?.claim_anchor_audit?.polarity_resolution_status ?? "not_applicable",
+        reason_codes: Array.isArray(debug?.grounded_answer_eligibility_guard?.reason_codes)
+          ? debug.grounded_answer_eligibility_guard.reason_codes
+          : [],
+        business_scope_resolved: getScopeResolved(debug),
+        admissible_evidence_count: getAdmissibleEvidenceCount(debug),
+        grounding_mode: getGroundingMode(debug),
+        debug
+      };
+      laneRows.push(row);
+      allRows.push(row);
+      writeJson(path.join(runDir, "debug_payloads", `${laneCase.case_id}_${laneCase.label}.json`), debug);
+    }
+    writeText(path.join(runDir, lane.chat_file), buildChatExport(laneRows, generatedAt));
+  }
+
+  const settlementPolarity = summarizeSettlementPolarity(allRows);
+  const settlementScope = summarizeSettlementScope(allRows);
+  const separation = summarizeLaneSeparation(allRows);
+  const falseGroundedRate = summarizeFalseGrounded(allRows);
+  const unresolvedPolarityCount = allRows.filter((item) =>
+    Array.isArray(item.reason_codes) && item.reason_codes.includes("polarity_guard_limited_unresolved_polarity")
+  ).length;
+  const unresolvedGenericScopeCount = allRows.filter((item) =>
+    Array.isArray(item.reason_codes) && item.reason_codes.includes("business_scope_generic_unresolved")
+  ).length;
+
+  const metrics = {
+    case_count: allRows.length,
+    settlement_role_resolution_rate: settlementPolarity.settlement_role_resolution_rate,
+    settlement_business_scope_resolution_rate: settlementScope.settlement_business_scope_resolution_rate,
+    wrong_family_route_rate: separation.wrong_family_route_rate,
+    wrong_claim_type_rate: separation.wrong_claim_type_rate,
+    month_close_lane_separation_rate: separation.month_close_lane_separation_rate,
+    mixed_family_acceptance_files_count: separation.mixed_family_acceptance_files_count,
+    false_grounded_answer_rate: falseGroundedRate,
+    unresolved_supplier_customer_polarity_count: unresolvedPolarityCount,
+    business_scope_generic_unresolved_count: unresolvedGenericScopeCount
+  };
+
+  const verdict = verdictFromMetrics(metrics);
+
+  const runSummary = {
+    stage: "Stage 4",
+    pack: "tail_pack_settlement_polarity_and_lane_separation",
+    generated_at: generatedAt,
+    use_mock: useMock,
+    prompt_version: promptVersion,
+    metrics,
+    verdict,
+    artifacts: {
+      run_summary: "run_summary.json",
+      settlement_polarity_report: "settlement_polarity_report.md",
+      settlement_scope_resolution_report: "settlement_scope_resolution_report.md",
+      lane_separation_report: "lane_separation_report.md",
+      family_acceptance_structure_report: "family_acceptance_structure_report.md",
+      chat_export_settlements: "chat_export_settlements.txt",
+      chat_export_rbp: "chat_export_rbp.txt",
+      chat_export_month_close_lane: "chat_export_month_close_lane.txt",
+      debug_payloads: "debug_payloads/"
+    }
+  };
+
+  const settlementPolarityReport = [
+    "# Settlement polarity report",
+    "",
+    `- total: ${settlementPolarity.total}`,
+    `- resolved: ${settlementPolarity.resolved}`,
+    `- unresolved: ${settlementPolarity.unresolved}`,
+    `- settlement_role_resolution_rate: ${settlementPolarity.settlement_role_resolution_rate}`,
+    `- unresolved_case_ids: ${settlementPolarity.unresolved_case_ids.join(", ") || "none"}`,
+    "",
+    "## Cases",
+    ...allRows.filter((item) => item.lane_id === "settlements_60_62").map(caseToExportLine)
+  ].join("\n");
+
+  const settlementScopeReport = [
+    "# Settlement scope resolution report",
+    "",
+    `- total: ${settlementScope.total}`,
+    `- resolved: ${settlementScope.resolved}`,
+    `- unresolved: ${settlementScope.unresolved}`,
+    `- settlement_business_scope_resolution_rate: ${settlementScope.settlement_business_scope_resolution_rate}`,
+    `- unresolved_case_ids: ${settlementScope.unresolved_case_ids.join(", ") || "none"}`
+  ].join("\n");
+
+  const laneSeparationReport = [
+    "# Lane separation report",
+    "",
+    `- wrong_family_route_rate: ${separation.wrong_family_route_rate}`,
+    `- wrong_claim_type_rate: ${separation.wrong_claim_type_rate}`,
+    `- month_close_lane_separation_rate: ${separation.month_close_lane_separation_rate}`,
+    `- mixed_family_acceptance_files_count: ${separation.mixed_family_acceptance_files_count}`,
+    `- wrong_case_ids: ${separation.wrong_case_ids.join(", ") || "none"}`,
+    "",
+    "## RBP lane claim types",
+    ...separation.rbp_lane_claims.map((item) => `- ${item.case_id}: ${item.claim_type || "n/a"}`),
+    "",
+    "## Month-close lane claim types",
+    ...separation.month_close_lane_claims.map((item) => `- ${item.case_id}: ${item.claim_type || "n/a"}`)
+  ].join("\n");
+
+  const acceptanceStructureReport = [
+    "# Family acceptance structure report",
+    "",
+    "- Rule: one acceptance file = one lane/family.",
+    "- chat_export_settlements.txt -> settlements_60_62 lane only.",
+    "- chat_export_rbp.txt -> rbp_tail lane only.",
+    "- chat_export_month_close_lane.txt -> month_close_indirect_costs lane only.",
+    "",
+    "## Verdict",
+    `- SETTLEMENT_POLARITY_FIXED: ${verdict.SETTLEMENT_POLARITY_FIXED}`,
+    `- SETTLEMENT_SCOPE_RESOLUTION_FIXED: ${verdict.SETTLEMENT_SCOPE_RESOLUTION_FIXED}`,
+    `- MONTH_CLOSE_RBP_LANE_SEPARATION_FIXED: ${verdict.MONTH_CLOSE_RBP_LANE_SEPARATION_FIXED}`,
+    `- overall_status: ${verdict.overall_status}`
+  ].join("\n");
+
+  writeJson(path.join(runDir, "run_summary.json"), runSummary);
+  writeText(path.join(runDir, "settlement_polarity_report.md"), `${settlementPolarityReport}\n`);
+  writeText(path.join(runDir, "settlement_scope_resolution_report.md"), `${settlementScopeReport}\n`);
+  writeText(path.join(runDir, "lane_separation_report.md"), `${laneSeparationReport}\n`);
+  writeText(path.join(runDir, "family_acceptance_structure_report.md"), `${acceptanceStructureReport}\n`);
+}
+
+main().catch((error) => {
+  process.stderr.write(`${error?.stack || error}\n`);
+  process.exitCode = 1;
+});