From df92bf9af2085cc45191ebcc8804e9fb5d50d27e Mon Sep 17 00:00:00 2001 From: dctouch Date: Thu, 23 Apr 2026 12:38:00 +0300 Subject: [PATCH] =?UTF-8?q?ARCH:=20=D0=B7=D0=B0=D0=BC=D0=BA=D0=BD=D1=83?= =?UTF-8?q?=D1=82=D1=8C=20multi-hop=20open=20total=20clarification=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...lti_hop_open_total_clarification_loop.json | 70 ++++++++++++++++ .../assistantMcpDiscoveryAnswerAdapter.js | 10 ++- .../assistantMcpDiscoveryResponseCandidate.js | 5 +- .../assistantMcpDiscoveryAnswerAdapter.ts | 10 ++- .../assistantMcpDiscoveryResponseCandidate.ts | 4 +- ...assistantMcpDiscoveryAnswerAdapter.test.ts | 45 +++++++++++ ...stantMcpDiscoveryResponseCandidate.test.ts | 30 +++++++ ...stantMcpDiscoveryRuntimeEntryPoint.test.ts | 81 +++++++++++++++++++ ...istantMcpDiscoveryTurnInputAdapter.test.ts | 37 +++++++++ 9 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 docs/orchestration/address_truth_harness_phase45_multi_hop_open_total_clarification_loop.json diff --git a/docs/orchestration/address_truth_harness_phase45_multi_hop_open_total_clarification_loop.json b/docs/orchestration/address_truth_harness_phase45_multi_hop_open_total_clarification_loop.json new file mode 100644 index 0000000..a69c934 --- /dev/null +++ b/docs/orchestration/address_truth_harness_phase45_multi_hop_open_total_clarification_loop.json @@ -0,0 +1,70 @@ +{ + "schema_version": "domain_truth_harness_spec_v1", + "scenario_id": "address_truth_harness_phase45_multi_hop_open_total_clarification_loop", + "domain": "address_phase45_multi_hop_open_total_clarification_loop", + "title": "Phase 45 multi-hop open total clarification loop", + "description": "Targeted AGENT replay for Big Block F where an open-scope incoming total must ask for both organization and period, then keep the same total loop after an organization-only clarification, and finally answer after the second clarification provides the period.", + "bindings": {}, + "steps": [ + { + "step_id": "step_01_open_total_requires_org_and_period", + "title": "Generic incoming total asks for both organization and period", + "question": "Сколько входящих денег?", + "allowed_reply_types": ["clarification_required", "partial_coverage", "factual_with_explanation"], + "required_answer_patterns_all": [ + "(?i)уточн|нужно", + "(?i)организац", + "(?i)период" + ], + "forbidden_answer_patterns": [ + "(?i)уточните контрагента", + "(?i)не найден контрагент", + "(?i)по какому контрагенту", + "(?i)не найдено контрагента" + ], + "criticality": "critical", + "semantic_tags": ["value_flow_total", "multi_hop_clarification", "organization_scope", "period_scope", "bounded_autonomy"] + }, + { + "step_id": "step_02_org_only_clarification_keeps_same_total_loop", + "title": "Organization-only clarification preserves the same total loop and asks only for the period", + "question": "по ООО Альтернатива Плюс", + "allowed_reply_types": ["clarification_required", "partial_coverage", "factual_with_explanation"], + "required_answer_patterns_all": [ + "(?i)уточн|нужно", + "(?i)период" + ], + "forbidden_answer_patterns": [ + "(?i)организац", + "(?i)уточните контрагента", + "(?i)не найден контрагент" + ], + "criticality": "critical", + "semantic_tags": ["value_flow_total", "multi_hop_clarification", "organization_followup_reuse", "bounded_autonomy"] + }, + { + "step_id": "step_03_period_clarification_completes_same_total_loop", + "title": "Period clarification completes the same total loop and yields a bounded incoming total answer", + "question": "за 2020 год", + "allowed_reply_types": ["factual_with_explanation", "partial_coverage"], + "required_answer_patterns_all": [ + "(?i)2020", + "(?i)входящ|получ|поступ", + "(?i)руб" + ], + "required_answer_patterns_any": [ + "(?i)подтвержден|проверенн|найденн", + "(?i)альтернатива", + "(?i)сумм" + ], + "forbidden_answer_patterns": [ + "(?i)уточните организацию", + "(?i)уточните период", + "(?i)уточните контрагента", + "(?i)не найден контрагент" + ], + "criticality": "critical", + "semantic_tags": ["value_flow_total", "multi_hop_clarification", "period_followup_reuse", "bounded_autonomy"] + } + ] +} diff --git a/llm_normalizer/backend/dist/services/assistantMcpDiscoveryAnswerAdapter.js b/llm_normalizer/backend/dist/services/assistantMcpDiscoveryAnswerAdapter.js index 61e54cf..c3a9676 100644 --- a/llm_normalizer/backend/dist/services/assistantMcpDiscoveryAnswerAdapter.js +++ b/llm_normalizer/backend/dist/services/assistantMcpDiscoveryAnswerAdapter.js @@ -45,7 +45,13 @@ function isInternalMechanicsLine(value) { text.includes("pilot_") || text.includes("runtime_") || text.includes("planner_") || - text.includes("catalog_")); + text.includes("catalog_") || + text.includes("mcp discovery") || + text.includes("needs more scope before execution") || + text.includes("mcp_execution_performed")); +} +function userFacingUnknowns(values) { + return uniqueStrings(values).filter((value) => !isInternalMechanicsLine(value)); } function userFacingLimitations(values) { return uniqueStrings(values).filter((value) => !isInternalMechanicsLine(value)); @@ -718,7 +724,7 @@ function buildAssistantMcpDiscoveryAnswerDraft(pilot) { headline: headlineFor(mode, pilot), confirmed_lines: uniqueStrings(confirmedLines), inference_lines: uniqueStrings(inferenceLines), - unknown_lines: uniqueStrings(pilot.evidence.unknown_facts), + unknown_lines: userFacingUnknowns(pilot.evidence.unknown_facts), limitation_lines: userFacingLimitations([...pilot.query_limitations, ...pilot.evidence.query_limitations]), next_step_line: nextStepFor(mode, pilot), internal_mechanics_allowed: false, diff --git a/llm_normalizer/backend/dist/services/assistantMcpDiscoveryResponseCandidate.js b/llm_normalizer/backend/dist/services/assistantMcpDiscoveryResponseCandidate.js index 3fa338a..8d6892f 100644 --- a/llm_normalizer/backend/dist/services/assistantMcpDiscoveryResponseCandidate.js +++ b/llm_normalizer/backend/dist/services/assistantMcpDiscoveryResponseCandidate.js @@ -55,7 +55,10 @@ function hasInternalMechanics(value) { text.includes("runtime_") || text.includes("planner_") || text.includes("catalog_") || - text.includes("select ")); + text.includes("select ") || + text.includes("mcp discovery") || + text.includes("needs more scope before execution") || + text.includes("mcp_execution_performed")); } function userFacingLines(values) { return uniqueStrings(values).filter((line) => !hasInternalMechanics(line)); diff --git a/llm_normalizer/backend/src/services/assistantMcpDiscoveryAnswerAdapter.ts b/llm_normalizer/backend/src/services/assistantMcpDiscoveryAnswerAdapter.ts index 60dd250..0de6b84 100644 --- a/llm_normalizer/backend/src/services/assistantMcpDiscoveryAnswerAdapter.ts +++ b/llm_normalizer/backend/src/services/assistantMcpDiscoveryAnswerAdapter.ts @@ -72,10 +72,16 @@ function isInternalMechanicsLine(value: string): boolean { text.includes("pilot_") || text.includes("runtime_") || text.includes("planner_") || - text.includes("catalog_") + text.includes("catalog_") || + text.includes("needs more scope before execution") || + text.includes("mcp_execution_performed") ); } +function userFacingUnknowns(values: string[]): string[] { + return uniqueStrings(values).filter((value) => !isInternalMechanicsLine(value)); +} + function userFacingLimitations(values: string[]): string[] { return uniqueStrings(values).filter((value) => !isInternalMechanicsLine(value)); } @@ -853,7 +859,7 @@ export function buildAssistantMcpDiscoveryAnswerDraft( headline: headlineFor(mode, pilot), confirmed_lines: uniqueStrings(confirmedLines), inference_lines: uniqueStrings(inferenceLines), - unknown_lines: uniqueStrings(pilot.evidence.unknown_facts), + unknown_lines: userFacingUnknowns(pilot.evidence.unknown_facts), limitation_lines: userFacingLimitations([...pilot.query_limitations, ...pilot.evidence.query_limitations]), next_step_line: nextStepFor(mode, pilot), internal_mechanics_allowed: false, diff --git a/llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts b/llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts index 6e86fd5..3827c29 100644 --- a/llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts +++ b/llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts @@ -82,7 +82,9 @@ function hasInternalMechanics(value: string): boolean { text.includes("runtime_") || text.includes("planner_") || text.includes("catalog_") || - text.includes("select ") + text.includes("select ") || + text.includes("needs more scope before execution") || + text.includes("mcp_execution_performed") ); } diff --git a/llm_normalizer/backend/tests/assistantMcpDiscoveryAnswerAdapter.test.ts b/llm_normalizer/backend/tests/assistantMcpDiscoveryAnswerAdapter.test.ts index ae38da2..ffe494d 100644 --- a/llm_normalizer/backend/tests/assistantMcpDiscoveryAnswerAdapter.test.ts +++ b/llm_normalizer/backend/tests/assistantMcpDiscoveryAnswerAdapter.test.ts @@ -296,6 +296,51 @@ describe("assistant MCP discovery answer adapter", () => { expect(draft.next_step_line).toContain("организац"); }); + it("asks for both organization and period when an open total still misses both axes", async () => { + const planner = planAssistantMcpDiscovery({ + dataNeedGraph: { + schema_version: "assistant_data_need_graph_v1", + policy_owner: "assistantMcpDiscoveryDataNeedGraph", + subject_candidates: [], + business_fact_family: "value_flow", + action_family: "turnover", + aggregation_need: null, + time_scope_need: "period_required", + comparison_need: null, + ranking_need: null, + proof_expectation: "clarification_required", + clarification_gaps: ["organization", "period"], + decomposition_candidates: ["collect_scoped_movements", "probe_coverage"], + forbidden_overclaim_flags: ["no_raw_model_claims", "no_unchecked_fact_totals"], + reason_codes: [ + "data_need_graph_built", + "data_need_graph_open_scope_total_without_subject", + "data_need_graph_open_scope_total_needs_organization", + "data_need_graph_has_clarification_gaps" + ] + }, + turnMeaning: { + asked_domain_family: "counterparty_value", + asked_action_family: "turnover" + } + }); + const pilot = await executeAssistantMcpDiscoveryPilot(planner, buildDeps([])); + + const draft = buildAssistantMcpDiscoveryAnswerDraft(pilot); + const periodToken = "\u043f\u0435\u0440\u0438\u043e\u0434"; + const organizationToken = "\u043e\u0440\u0433\u0430\u043d\u0438\u0437\u0430\u0446"; + const counterpartyToken = "\u043a\u043e\u043d\u0442\u0440\u0430\u0433\u0435\u043d\u0442"; + + expect(draft.answer_mode).toBe("needs_clarification"); + expect(draft.headline).toContain(periodToken); + expect(draft.headline).toContain(organizationToken); + expect(draft.next_step_line).toContain(periodToken); + expect(draft.next_step_line).toContain(organizationToken); + expect(draft.next_step_line).not.toContain(counterpartyToken); + expect(draft.unknown_lines).toEqual([]); + expect(draft.limitation_lines).toEqual([]); + }); + it("asks for organization rather than counterparty on open bidirectional comparison when only the period is known", async () => { const planner = planAssistantMcpDiscovery({ dataNeedGraph: { diff --git a/llm_normalizer/backend/tests/assistantMcpDiscoveryResponseCandidate.test.ts b/llm_normalizer/backend/tests/assistantMcpDiscoveryResponseCandidate.test.ts index be3921f..88547df 100644 --- a/llm_normalizer/backend/tests/assistantMcpDiscoveryResponseCandidate.test.ts +++ b/llm_normalizer/backend/tests/assistantMcpDiscoveryResponseCandidate.test.ts @@ -434,4 +434,34 @@ describe("assistant MCP discovery response candidate", () => { ); expect(candidate.reply_text).not.toContain("Requested period hit the MCP row limit"); }); + + it("filters MCP discovery scope-mechanics from clarification unknown and limitation blocks", () => { + const candidate = buildAssistantMcpDiscoveryResponseCandidate( + entryPoint({ + bridge: { + bridge_status: "needs_clarification", + user_facing_response_allowed: true, + business_fact_answer_allowed: false, + requires_user_clarification: true, + answer_draft: { + answer_mode: "needs_clarification", + headline: + "Могу посчитать общий денежный поток в проверяемом окне, но для проверяемого поиска в 1С нужно проверяемый период и организацию.", + confirmed_lines: [], + inference_lines: [], + unknown_lines: ["MCP discovery pilot needs more scope before execution"], + limitation_lines: ["MCP discovery pilot needs more scope before execution"], + next_step_line: + "Уточните период и организацию, и я продолжу поиск по денежному потоку в 1С." + } + } + }) + ); + + expect(candidate.reply_text).toContain("Уточните период и организацию"); + expect(candidate.reply_text).not.toContain("MCP discovery"); + expect(candidate.reply_text).not.toContain("needs more scope before execution"); + expect(candidate.reply_text).not.toContain("Что не подтверждено:"); + expect(candidate.reply_text).not.toContain("Ограничения проверки:"); + }); }); diff --git a/llm_normalizer/backend/tests/assistantMcpDiscoveryRuntimeEntryPoint.test.ts b/llm_normalizer/backend/tests/assistantMcpDiscoveryRuntimeEntryPoint.test.ts index 0eb0025..47537f3 100644 --- a/llm_normalizer/backend/tests/assistantMcpDiscoveryRuntimeEntryPoint.test.ts +++ b/llm_normalizer/backend/tests/assistantMcpDiscoveryRuntimeEntryPoint.test.ts @@ -408,6 +408,87 @@ describe("assistant MCP discovery runtime entry point", () => { expect(result.bridge?.loop_state.pending_axes).toEqual([]); }); + it("keeps the same open total loop after organization-only clarification when period is still missing", async () => { + const periodToken = "\u043f\u0435\u0440\u0438\u043e\u0434"; + const organizationToken = "\u043e\u0440\u0433\u0430\u043d\u0438\u0437\u0430\u0446"; + const result = await runAssistantMcpDiscoveryRuntimeEntryPoint({ + userMessage: "РїРѕ РћРћРћ Альтернатива Плюс", + predecomposeContract: { + entities: { organization: "РћРћРћ Альтернатива Плюс" } + }, + followupContext: { + previous_discovery_loop_status: "awaiting_clarification", + previous_discovery_loop_selected_chain_id: "value_flow", + previous_discovery_loop_pending_axes: ["organization", "period"], + previous_discovery_loop_provided_axes: ["amount", "coverage_target"], + previous_discovery_loop_asked_domain_family: "counterparty_value", + previous_discovery_loop_asked_action_family: "turnover", + previous_discovery_loop_unsupported_family: "counterparty_value_or_turnover" + }, + deps: buildDeps([]) + }); + + expect(result.entry_status).toBe("bridge_executed"); + expect(result.turn_input.turn_meaning_ref).toMatchObject({ + asked_domain_family: "counterparty_value", + asked_action_family: "turnover", + explicit_organization_scope: "РћРћРћ Альтернатива Плюс" + }); + expect(result.turn_input.turn_meaning_ref?.explicit_date_scope).toBeUndefined(); + expect(result.bridge?.bridge_status).toBe("needs_clarification"); + expect(result.bridge?.planner.selected_chain_id).toBe("value_flow"); + expect(result.bridge?.loop_state).toMatchObject({ + loop_status: "awaiting_clarification", + selected_chain_id: "value_flow", + explicit_organization_scope: "РћРћРћ Альтернатива Плюс", + explicit_date_scope: null + }); + expect(result.bridge?.loop_state.pending_axes).toContain("period"); + expect(result.bridge?.loop_state.pending_axes).not.toContain("organization"); + expect(result.bridge?.answer_draft.next_step_line).toContain(periodToken); + expect(result.bridge?.answer_draft.next_step_line).not.toContain(organizationToken); + }); + + it("completes the same open total loop after the second clarification provides the period", async () => { + const result = await runAssistantMcpDiscoveryRuntimeEntryPoint({ + userMessage: "Р·Р° 2020 РіРѕРґ", + followupContext: { + previous_discovery_loop_status: "awaiting_clarification", + previous_discovery_loop_selected_chain_id: "value_flow", + previous_discovery_loop_pending_axes: ["period"], + previous_discovery_loop_provided_axes: ["amount", "coverage_target", "organization"], + previous_discovery_loop_asked_domain_family: "counterparty_value", + previous_discovery_loop_asked_action_family: "turnover", + previous_discovery_loop_unsupported_family: "counterparty_value_or_turnover", + previous_filters: { + organization: "РћРћРћ Альтернатива Плюс" + } + }, + deps: buildDeps([ + { Period: "2020-01-10T00:00:00", Amount: 1200, Counterparty: "РЎР’Рљ-Рђ" }, + { Period: "2020-03-11T00:00:00", Amount: 800, Counterparty: "РЎР’Рљ-Р‘" }, + { Period: "2020-05-12T00:00:00", Amount: 900, Counterparty: "РЎР’Рљ-Рђ" } + ]) + }); + + expect(result.entry_status).toBe("bridge_executed"); + expect(result.turn_input.turn_meaning_ref).toMatchObject({ + asked_domain_family: "counterparty_value", + asked_action_family: "turnover", + explicit_organization_scope: "РћРћРћ Альтернатива Плюс", + explicit_date_scope: "2020" + }); + expect(result.bridge?.bridge_status).toBe("answer_draft_ready"); + expect(result.bridge?.planner.selected_chain_id).toBe("value_flow"); + expect(result.bridge?.pilot.derived_value_flow).toMatchObject({ + counterparty: null, + period_scope: "2020", + total_amount: 2900 + }); + expect(result.bridge?.loop_state.loop_status).toBe("ready_for_next_hop"); + expect(result.bridge?.loop_state.pending_axes).toEqual([]); + }); + it.skip("keeps mirrored predecompose organization and counterparty out of the subject lane for open comparison (utf8-safe)", async () => { const orgName = "\u041e\u041e\u041e \u0410\u043b\u044c\u0442\u0435\u0440\u043d\u0430\u0442\u0438\u0432\u0430 \u041f\u043b\u044e\u0441"; const result = await runAssistantMcpDiscoveryRuntimeEntryPoint({ diff --git a/llm_normalizer/backend/tests/assistantMcpDiscoveryTurnInputAdapter.test.ts b/llm_normalizer/backend/tests/assistantMcpDiscoveryTurnInputAdapter.test.ts index e85072e..41f922c 100644 --- a/llm_normalizer/backend/tests/assistantMcpDiscoveryTurnInputAdapter.test.ts +++ b/llm_normalizer/backend/tests/assistantMcpDiscoveryTurnInputAdapter.test.ts @@ -1678,6 +1678,43 @@ describe("assistant MCP discovery turn input adapter", () => { expect(result.data_need_graph?.clarification_gaps).toEqual(["period"]); }); + it("does not keep an implicit today date while an open total clarification loop still needs period", () => { + const todayIso = new Date().toISOString().slice(0, 10); + const orgName = "РћРћРћ Альтернатива Плюс"; + const result = buildAssistantMcpDiscoveryTurnInput({ + userMessage: "РїРѕ РћРћРћ Альтернатива Плюс", + assistantTurnMeaning: { + asked_domain_family: "counterparty_value", + asked_action_family: "turnover", + explicit_date_scope: todayIso, + explicit_intent_candidate: "customer_revenue_and_payments" + }, + predecomposeContract: { + entities: { organization: orgName } + }, + followupContext: { + previous_discovery_loop_status: "awaiting_clarification", + previous_discovery_loop_selected_chain_id: "value_flow", + previous_discovery_loop_pending_axes: ["organization", "period"], + previous_discovery_loop_provided_axes: ["amount", "coverage_target"], + previous_discovery_loop_asked_domain_family: "counterparty_value", + previous_discovery_loop_asked_action_family: "turnover", + previous_discovery_loop_unsupported_family: "counterparty_value_or_turnover", + previous_filters: { + period_to: todayIso + } + } + }); + + expect(result.turn_meaning_ref).toMatchObject({ + asked_domain_family: "counterparty_value", + asked_action_family: "turnover", + explicit_organization_scope: orgName + }); + expect(result.turn_meaning_ref?.explicit_date_scope).toBeUndefined(); + expect(result.data_need_graph?.clarification_gaps).toEqual(["period"]); + }); + it("resolves metadata lane choice from saved loop state even without a previous pilot scope", () => { const result = buildAssistantMcpDiscoveryTurnInput({ userMessage: "по движениям",