Очеловечить MCP ответы и broad-eval recap
This commit is contained in:
@@ -407,6 +407,53 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
expect(draft.unknown_lines[0]).toContain("\u041f\u043e\u043b\u043d\u044b\u0439 \u0440\u0435\u0439\u0442\u0438\u043d\u0433");
|
||||
});
|
||||
|
||||
it("does not overclaim a comparative ranking when only one counterparty is present", async () => {
|
||||
const planner = planAssistantMcpDiscovery({
|
||||
dataNeedGraph: {
|
||||
schema_version: "assistant_data_need_graph_v1",
|
||||
policy_owner: "assistantMcpDiscoveryDataNeedGraph",
|
||||
subject_candidates: [],
|
||||
business_fact_family: "value_flow",
|
||||
action_family: "turnover",
|
||||
aggregation_need: null,
|
||||
time_scope_need: "explicit_period",
|
||||
comparison_need: null,
|
||||
ranking_need: "top_desc",
|
||||
proof_expectation: "coverage_checked_fact",
|
||||
clarification_gaps: [],
|
||||
decomposition_candidates: ["collect_scoped_movements", "aggregate_ranked_axis_values", "probe_coverage"],
|
||||
forbidden_overclaim_flags: ["no_raw_model_claims", "no_unchecked_fact_totals"],
|
||||
reason_codes: ["data_need_graph_built", "data_need_graph_ranking_top_desc"]
|
||||
},
|
||||
turnMeaning: {
|
||||
asked_domain_family: "counterparty_value",
|
||||
asked_action_family: "turnover",
|
||||
explicit_organization_scope: "ООО Альтернатива Плюс",
|
||||
explicit_date_scope: "2021",
|
||||
seeded_ranking_need: "top_desc"
|
||||
}
|
||||
});
|
||||
const pilot = await executeAssistantMcpDiscoveryPilot(
|
||||
planner,
|
||||
buildDeps([
|
||||
{
|
||||
Period: "2021-01-15T00:00:00",
|
||||
Amount: 8560025,
|
||||
Counterparty: "Группа СВК",
|
||||
Organization: "ООО Альтернатива Плюс"
|
||||
}
|
||||
])
|
||||
);
|
||||
|
||||
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
|
||||
const text = draft.confirmed_lines.join("\n");
|
||||
|
||||
expect(text).toContain("найден один контрагент");
|
||||
expect(text).toContain("Группа СВК");
|
||||
expect(text).toContain("не полноценный сравнительный рейтинг");
|
||||
expect(text).not.toContain("Больше всего денег принёс");
|
||||
});
|
||||
|
||||
it("asks for both organization and period when an open total still misses both axes", async () => {
|
||||
const planner = planAssistantMcpDiscovery({
|
||||
dataNeedGraph: {
|
||||
@@ -501,7 +548,7 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
|
||||
|
||||
expect(draft.answer_mode).toBe("needs_clarification");
|
||||
expect(draft.headline).toContain("data-lane");
|
||||
expect(draft.headline).toContain("контуров");
|
||||
expect(draft.next_step_line).toContain("по документам");
|
||||
expect(draft.next_step_line).toContain("по движениям/регистрам");
|
||||
expect(draft.must_not_claim).toContain("Do not claim rows were checked when mcp_execution_performed=false.");
|
||||
@@ -537,7 +584,7 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
|
||||
|
||||
expect(draft.answer_mode).toBe("needs_clarification");
|
||||
expect(draft.headline).toContain("data-lane");
|
||||
expect(draft.headline).toContain("контуров");
|
||||
expect(draft.next_step_line).toContain("по документам");
|
||||
expect(draft.next_step_line).toContain("по движениям/регистрам");
|
||||
expect(draft.next_step_line).not.toContain("Уточните контрагента");
|
||||
@@ -671,13 +718,15 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
const confirmedText = draft.confirmed_lines.join("\n");
|
||||
|
||||
expect(draft.answer_mode).toBe("confirmed_with_bounded_inference");
|
||||
expect(draft.headline).toContain("заземлена вероятная поверхность");
|
||||
expect(confirmedText).toContain("Подтвержденная metadata-поверхность 1С");
|
||||
expect(draft.headline).toContain("схеме 1С");
|
||||
expect(confirmedText).toContain("В схеме 1С");
|
||||
expect(confirmedText).toContain("Документ.СчетФактураВыданный");
|
||||
expect(confirmedText).toContain("Выбранное family: Документ");
|
||||
expect(confirmedText).toContain("Для следующего шага подходят");
|
||||
expect(confirmedText).toContain("Дата");
|
||||
expect(draft.inference_lines.join("\n")).toContain("контур документов");
|
||||
expect(draft.next_step_line).toContain("surface «Документ»");
|
||||
expect(draft.next_step_line).toContain("типу «Документ»");
|
||||
expect(confirmedText).not.toContain("Confirmed 1C metadata surface");
|
||||
expect(confirmedText).not.toContain("Metadata surface family scores");
|
||||
expect(draft.must_not_claim).toContain("Do not present metadata surface as confirmed business data rows.");
|
||||
expect(draft.must_not_claim).toContain("Do not present the inferred next checked lane as already executed data retrieval.");
|
||||
});
|
||||
@@ -715,7 +764,42 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
expect(draft.answer_mode).toBe("confirmed_with_bounded_inference");
|
||||
expect(draft.headline).toContain("углубиться");
|
||||
expect(draft.headline).toContain("справочников");
|
||||
expect(draft.headline).toContain("catalog drilldown");
|
||||
expect(draft.headline).not.toContain("catalog drilldown");
|
||||
});
|
||||
|
||||
it("renders catalog metadata without leaking internal surface scoring", async () => {
|
||||
const planner = planAssistantMcpDiscovery({
|
||||
turnMeaning: {
|
||||
asked_domain_family: "metadata",
|
||||
asked_action_family: "inspect_catalog",
|
||||
explicit_entity_candidates: ["контрагент"],
|
||||
unsupported_but_understood_family: "1c_metadata_surface"
|
||||
}
|
||||
});
|
||||
const pilot = await executeAssistantMcpDiscoveryPilot(
|
||||
planner,
|
||||
buildMetadataDeps([
|
||||
{ FullName: "Справочник.ДоговорыКонтрагентов", MetaType: "Справочник" },
|
||||
{ FullName: "Справочник.Контрагенты", MetaType: "Справочник" }
|
||||
])
|
||||
);
|
||||
|
||||
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
|
||||
const userText = [
|
||||
draft.headline,
|
||||
...draft.confirmed_lines,
|
||||
...draft.inference_lines,
|
||||
...draft.unknown_lines,
|
||||
draft.next_step_line ?? ""
|
||||
].join("\n");
|
||||
|
||||
expect(userText).toContain("Справочник.ДоговорыКонтрагентов");
|
||||
expect(userText).toContain("Справочник.Контрагенты");
|
||||
expect(userText).toContain("Детальный список полей");
|
||||
expect(userText).not.toContain("Confirmed 1C metadata surface");
|
||||
expect(userText).not.toContain("Metadata surface family scores");
|
||||
expect(userText).not.toContain("surface «");
|
||||
expect(userText).not.toContain("catalog drilldown");
|
||||
});
|
||||
|
||||
it("keeps metadata answer honest when schema surface stays ambiguous", async () => {
|
||||
@@ -744,8 +828,8 @@ describe("assistant MCP discovery answer adapter", () => {
|
||||
|
||||
const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot);
|
||||
|
||||
expect(draft.headline).toContain("конкурирующие schema-поверхности");
|
||||
expect(draft.inference_lines.join("\n")).toContain("несколько конкурирующих family");
|
||||
expect(draft.headline).toContain("конкурирующих контуров");
|
||||
expect(draft.inference_lines.join("\n")).toContain("несколько возможных контуров");
|
||||
expect(draft.unknown_lines).toContain(
|
||||
"Exact downstream metadata surface remains ambiguous across: Документ, РегистрНакопления"
|
||||
);
|
||||
|
||||
@@ -439,7 +439,7 @@ describe("assistantMemoryRecapPolicy", () => {
|
||||
|
||||
expect(context.contextualMemoryRecapFollowup).toBe(true);
|
||||
expect(reply).toContain("НДС");
|
||||
expect(reply).toContain("metadata-поверхность 1С");
|
||||
expect(reply).toContain("схему 1С");
|
||||
expect(reply).toContain("amount");
|
||||
expect(reply).toContain("accumulation_register");
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user