Очеловечить MCP ответы и broad-eval recap

This commit is contained in:
2026-05-01 22:38:23 +03:00
parent 4c00d8c854
commit da3a148918
6 changed files with 295 additions and 73 deletions
@@ -407,6 +407,53 @@ describe("assistant MCP discovery answer adapter", () => {
expect(draft.unknown_lines[0]).toContain("\u041f\u043e\u043b\u043d\u044b\u0439 \u0440\u0435\u0439\u0442\u0438\u043d\u0433");
});
it("does not overclaim a comparative ranking when only one counterparty is present", async () => {
const planner = planAssistantMcpDiscovery({
dataNeedGraph: {
schema_version: "assistant_data_need_graph_v1",
policy_owner: "assistantMcpDiscoveryDataNeedGraph",
subject_candidates: [],
business_fact_family: "value_flow",
action_family: "turnover",
aggregation_need: null,
time_scope_need: "explicit_period",
comparison_need: null,
ranking_need: "top_desc",
proof_expectation: "coverage_checked_fact",
clarification_gaps: [],
decomposition_candidates: ["collect_scoped_movements", "aggregate_ranked_axis_values", "probe_coverage"],
forbidden_overclaim_flags: ["no_raw_model_claims", "no_unchecked_fact_totals"],
reason_codes: ["data_need_graph_built", "data_need_graph_ranking_top_desc"]
},
turnMeaning: {
asked_domain_family: "counterparty_value",
asked_action_family: "turnover",
explicit_organization_scope: "ООО Альтернатива Плюс",
explicit_date_scope: "2021",
seeded_ranking_need: "top_desc"
}
});
const pilot = await executeAssistantMcpDiscoveryPilot(
planner,
buildDeps([
{
Period: "2021-01-15T00:00:00",
Amount: 8560025,
Counterparty: "Группа СВК",
Organization: "ООО Альтернатива Плюс"
}
])
);
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
const text = draft.confirmed_lines.join("\n");
expect(text).toContain("найден один контрагент");
expect(text).toContain("Группа СВК");
expect(text).toContain("не полноценный сравнительный рейтинг");
expect(text).not.toContain("Больше всего денег принёс");
});
it("asks for both organization and period when an open total still misses both axes", async () => {
const planner = planAssistantMcpDiscovery({
dataNeedGraph: {
@@ -501,7 +548,7 @@ describe("assistant MCP discovery answer adapter", () => {
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
expect(draft.answer_mode).toBe("needs_clarification");
expect(draft.headline).toContain("data-lane");
expect(draft.headline).toContain("контуров");
expect(draft.next_step_line).toContain("по документам");
expect(draft.next_step_line).toContain("по движениям/регистрам");
expect(draft.must_not_claim).toContain("Do not claim rows were checked when mcp_execution_performed=false.");
@@ -537,7 +584,7 @@ describe("assistant MCP discovery answer adapter", () => {
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
expect(draft.answer_mode).toBe("needs_clarification");
expect(draft.headline).toContain("data-lane");
expect(draft.headline).toContain("контуров");
expect(draft.next_step_line).toContain("по документам");
expect(draft.next_step_line).toContain("по движениям/регистрам");
expect(draft.next_step_line).not.toContain("Уточните контрагента");
@@ -671,13 +718,15 @@ describe("assistant MCP discovery answer adapter", () => {
const confirmedText = draft.confirmed_lines.join("\n");
expect(draft.answer_mode).toBe("confirmed_with_bounded_inference");
expect(draft.headline).toContain("заземлена вероятная поверхность");
expect(confirmedText).toContain("Подтвержденная metadata-поверхность 1С");
expect(draft.headline).toContain("схеме 1С");
expect(confirmedText).toContain("В схеме 1С");
expect(confirmedText).toContain("Документ.СчетФактураВыданный");
expect(confirmedText).toContain("Выбранное family: Документ");
expect(confirmedText).toContain("Для следующего шага подходят");
expect(confirmedText).toContain("Дата");
expect(draft.inference_lines.join("\n")).toContain("контур документов");
expect(draft.next_step_line).toContain("surface «Документ»");
expect(draft.next_step_line).toContain("типу «Документ»");
expect(confirmedText).not.toContain("Confirmed 1C metadata surface");
expect(confirmedText).not.toContain("Metadata surface family scores");
expect(draft.must_not_claim).toContain("Do not present metadata surface as confirmed business data rows.");
expect(draft.must_not_claim).toContain("Do not present the inferred next checked lane as already executed data retrieval.");
});
@@ -715,7 +764,42 @@ describe("assistant MCP discovery answer adapter", () => {
expect(draft.answer_mode).toBe("confirmed_with_bounded_inference");
expect(draft.headline).toContain("углубиться");
expect(draft.headline).toContain("справочников");
expect(draft.headline).toContain("catalog drilldown");
expect(draft.headline).not.toContain("catalog drilldown");
});
it("renders catalog metadata without leaking internal surface scoring", async () => {
const planner = planAssistantMcpDiscovery({
turnMeaning: {
asked_domain_family: "metadata",
asked_action_family: "inspect_catalog",
explicit_entity_candidates: ["контрагент"],
unsupported_but_understood_family: "1c_metadata_surface"
}
});
const pilot = await executeAssistantMcpDiscoveryPilot(
planner,
buildMetadataDeps([
{ FullName: "Справочник.ДоговорыКонтрагентов", MetaType: "Справочник" },
{ FullName: "Справочник.Контрагенты", MetaType: "Справочник" }
])
);
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
const userText = [
draft.headline,
...draft.confirmed_lines,
...draft.inference_lines,
...draft.unknown_lines,
draft.next_step_line ?? ""
].join("\n");
expect(userText).toContain("Справочник.ДоговорыКонтрагентов");
expect(userText).toContain("Справочник.Контрагенты");
expect(userText).toContain("Детальный список полей");
expect(userText).not.toContain("Confirmed 1C metadata surface");
expect(userText).not.toContain("Metadata surface family scores");
expect(userText).not.toContain("surface «");
expect(userText).not.toContain("catalog drilldown");
});
it("keeps metadata answer honest when schema surface stays ambiguous", async () => {
@@ -744,8 +828,8 @@ describe("assistant MCP discovery answer adapter", () => {
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
expect(draft.headline).toContain("конкурирующие schema-поверхности");
expect(draft.inference_lines.join("\n")).toContain("несколько конкурирующих family");
expect(draft.headline).toContain("конкурирующих контуров");
expect(draft.inference_lines.join("\n")).toContain("несколько возможных контуров");
expect(draft.unknown_lines).toContain(
"Exact downstream metadata surface remains ambiguous across: Документ, РегистрНакопления"
);
@@ -439,7 +439,7 @@ describe("assistantMemoryRecapPolicy", () => {
expect(context.contextualMemoryRecapFollowup).toBe(true);
expect(reply).toContain("НДС");
expect(reply).toContain("metadata-поверхность 1С");
expect(reply).toContain("схему 1С");
expect(reply).toContain("amount");
expect(reply).toContain("accumulation_register");
});