From 2873758579d989d893c87e2066ffe95bed7e395a Mon Sep 17 00:00:00 2001 From: dctouch Date: Mon, 15 Jun 2026 23:47:04 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B2=D0=B0=D0=BB=D0=B8=D0=B4=D0=B0=D1=86=D0=B8=D1=8E?= =?UTF-8?q?=20guarded=20MCP=20=D0=BE=D1=82=D0=B2=D0=B5=D1=82=D0=BE=D0=B2?= =?UTF-8?q?=20=D0=B0=D0=B3=D0=B5=D0=BD=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/agent_detector_runner.py | 18 +++++-- scripts/domain_case_loop.py | 36 +++++++++++-- scripts/test_agent_detector_runner.py | 59 +++++++++++++++++++++ scripts/test_domain_case_loop_step_state.py | 40 ++++++++++++++ 4 files changed, 146 insertions(+), 7 deletions(-) diff --git a/scripts/agent_detector_runner.py b/scripts/agent_detector_runner.py index 641d9a3..582b7f0 100644 --- a/scripts/agent_detector_runner.py +++ b/scripts/agent_detector_runner.py @@ -518,12 +518,24 @@ def evaluate_limited_next_action( limited_outputs = 0 for output in outputs: text = current_answer_text_for_output(output) + step_state_path = output.get("step_state_path") + step_state = read_json_object(step_state_path) if isinstance(step_state_path, Path) else {} + review = step_state.get("business_first_review") if isinstance(step_state.get("business_first_review"), dict) else {} + review_limited = review.get("limited_answer_detected") + if review_limited is False: + continue limited_match = first_pattern_search(limited_patterns, text) - if not limited_match: + if review_limited is not True and not limited_match: continue limited_outputs += 1 - if not first_pattern_search(next_action_patterns, text): - failures.append({"path": output["repo_path"], "limited_match": limited_match.group(0)[:240]}) + next_action_present = review.get("next_action_present") is True or bool(first_pattern_search(next_action_patterns, text)) + if not next_action_present: + evidence = {"path": output["repo_path"]} + if limited_match: + evidence["limited_match"] = limited_match.group(0)[:240] + else: + evidence["limited_match"] = "business_first_review.limited_answer_detected" + failures.append(evidence) status = "fail" if failures else "pass" if failures: message = "limited answer has no next action" diff --git a/scripts/domain_case_loop.py b/scripts/domain_case_loop.py index bb0fcfa..b16acd4 100644 --- a/scripts/domain_case_loop.py +++ b/scripts/domain_case_loop.py @@ -260,6 +260,10 @@ GUARDED_INSUFFICIENCY_PRIMARY_MARKERS = ( "\u043d\u0435 \u043f\u043e\u0434\u0442\u0432\u0435\u0440\u0436\u0434\u0430", "\u043d\u0435\u043b\u044c\u0437\u044f \u0447\u0435\u0441\u0442\u043d\u043e \u043f\u043e\u0434\u0442\u0432\u0435\u0440\u0434", "\u043d\u0435\u043b\u044c\u0437\u044f \u043a\u043e\u0440\u0440\u0435\u043a\u0442\u043d\u043e \u043e\u043f\u0440\u0435\u0434\u0435\u043b", + "\u043d\u0435\u043b\u044c\u0437\u044f \u0434\u043e\u043a\u0430\u0437\u0430\u0442\u044c", + "\u043d\u0435 \u0434\u043e\u043a\u0430\u0437\u0430\u0442\u044c", + "\u043d\u0435 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043b\u0435\u043d \u0441\u0440\u043e\u043a \u043e\u043f\u043b\u0430\u0442\u044b", + "\u043d\u0435\u0442 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043b\u0435\u043d\u043d\u043e\u0433\u043e \u0441\u0440\u043e\u043a\u0430 \u043e\u043f\u043b\u0430\u0442\u044b", "\u0447\u0435\u0441\u0442\u043d\u043e \u043f\u043e\u0441\u0447\u0438\u0442\u0430\u0442\u044c \u043d\u0435\u043b\u044c\u0437\u044f", "\u043d\u0435\u0442 \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e\u0439 \u0431\u0430\u0437\u044b", "\u043f\u043e\u0434\u0442\u0432\u0435\u0440\u0436\u0434\u0435\u043d\u043d\u043e\u0439 \u0441\u0435\u0431\u0435\u0441\u0442\u043e\u0438\u043c", @@ -275,6 +279,9 @@ GUARDED_INSUFFICIENCY_LIMITATION_MARKERS = ( "\u043d\u0435 \u0434\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u043e\u0441\u0442\u0430\u0442\u043e\u043a", "\u043d\u0435 \u0444\u0438\u043d\u0430\u043b\u044c\u043d\u044b\u0439 \u0440\u0435\u0435\u0441\u0442\u0440", "\u0433\u0440\u0430\u043d\u0438\u0446\u0430 \u043e\u0442\u0432\u0435\u0442\u0430", + "due date", + "\u0441\u0440\u043e\u043a \u043e\u043f\u043b\u0430\u0442\u044b", + "\u0434\u043e\u0433\u043e\u0432\u043e\u0440\u043d\u043e\u0439 \u043f\u0440\u043e\u0441\u0440\u043e\u0447", "\u0440\u0430\u0441\u0447\u0435\u0442\u043d\u0443\u044e \u0431\u0430\u0437\u0443", "\u0440\u0430\u0441\u0447\u0451\u0442\u043d\u0443\u044e \u0431\u0430\u0437\u0443", ) @@ -2798,6 +2805,26 @@ def has_guarded_insufficiency_language(answer_text: str) -> bool: return has_unconfirmed_marker and has_limitation_marker +def is_validated_guarded_mcp_discovery_insufficiency(state: dict[str, Any]) -> bool: + if state.get("mcp_discovery_response_applied") is not True: + return False + if str(state.get("mcp_discovery_response_candidate_status") or "").strip() != "ready_for_guarded_use": + return False + if str(state.get("mcp_discovery_route_candidate_status") or "").strip() != "ready_for_reviewed_execution": + return False + if str(state.get("mcp_discovery_execution_handoff_can_use_guarded_response") or "").casefold() not in {"true", "1"}: + return False + if str(state.get("truth_gate_contract_status") or "").strip() != "partial_supported": + return False + if str(state.get("truth_mode") or "").strip() not in GUARDED_INSUFFICIENCY_TRUTH_MODES: + return False + if str(state.get("answer_shape") or "").strip() not in GUARDED_INSUFFICIENCY_ANSWER_SHAPES: + return False + if str(state.get("evidence_grade") or "").strip() not in {"weak", "medium", "strong"}: + return False + return bool(str(state.get("assistant_text") or "").strip()) + + def is_validated_guarded_insufficiency_answer( state: dict[str, Any], execution_status: str, @@ -2808,13 +2835,14 @@ def is_validated_guarded_insufficiency_answer( return False if violations: return False - if state.get("mcp_discovery_response_applied") is True: + mcp_discovery_guarded = is_validated_guarded_mcp_discovery_insufficiency(state) + if state.get("mcp_discovery_response_applied") is True and not mcp_discovery_guarded: return False if str(state.get("reply_type") or "").strip() not in {"factual", "factual_with_explanation", "partial_coverage"}: return False - if str(state.get("fallback_type") or "").strip() not in {"", "none"}: + if not mcp_discovery_guarded and str(state.get("fallback_type") or "").strip() not in {"", "none"}: return False - if str(state.get("mcp_call_status") or "").strip() != "matched_non_empty": + if not mcp_discovery_guarded and str(state.get("mcp_call_status") or "").strip() != "matched_non_empty": return False result_mode = str(state.get("result_mode") or "").strip() truth_mode = str(state.get("truth_mode") or "").strip() @@ -2825,7 +2853,7 @@ def is_validated_guarded_insufficiency_answer( or answer_shape in GUARDED_INSUFFICIENCY_ANSWER_SHAPES or state.get("balance_confirmed") is False ) - if not has_limited_evidence_mode: + if not (has_limited_evidence_mode or mcp_discovery_guarded): return False if not has_guarded_insufficiency_language(str(state.get("assistant_text") or "")): return False diff --git a/scripts/test_agent_detector_runner.py b/scripts/test_agent_detector_runner.py index 657a117..fd67314 100644 --- a/scripts/test_agent_detector_runner.py +++ b/scripts/test_agent_detector_runner.py @@ -160,6 +160,65 @@ class AgentDetectorRunnerTests(unittest.TestCase): self.assertEqual(results["summary"]["status"], "pass") self.assertEqual(results["results"][0]["message"], "limited answer includes a next action") + def test_limited_next_action_ignores_non_limited_off_domain_can_not_phrase(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact_dir = root / "run" + step_dir = artifact_dir / "scenarios" / "chat" / "steps" / "s01" + write_text(step_dir / "output.md", "Нет, пингвины не могут летать.") + write_json( + step_dir / "step_state.json", + { + "business_first_review": { + "limited_answer_detected": False, + "next_action_present": False, + } + }, + ) + registry_path = root / "detector_registry.json" + issue_catalog_path = root / "issue_catalog.json" + write_json( + registry_path, + { + "schema_version": "agent_detector_registry_v1", + "detectors": { + "limited_answer_without_next_action": { + "kind": "answer_text_required_when_limited", + "automation_level": "semi_automatic", + "description": "Limited answers need a next action.", + "issue_codes": ["business_next_step_missing"], + "inputs": ["output.md"], + "check": { + "limited_patterns": ["(?i)не могу"], + "required_next_action_patterns_any": ["(?i)что проверить дальше"], + }, + } + }, + }, + ) + write_json( + issue_catalog_path, + { + "schema_version": "agent_issue_catalog_v1", + "issues": { + "business_next_step_missing": { + "detectors": ["limited_answer_without_next_action"], + } + }, + }, + ) + + results = runner.build_detector_results( + artifact_dir, + issue_codes=["business_next_step_missing"], + registry_path=registry_path, + issue_catalog_path=issue_catalog_path, + include_default_global=False, + ) + + self.assertEqual(results["summary"]["status"], "pass") + self.assertEqual(results["results"][0]["message"], "answer is not limited; next action requirement not triggered") + def test_candidate_forbidden_regex_is_limited_to_evidence_path(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/scripts/test_domain_case_loop_step_state.py b/scripts/test_domain_case_loop_step_state.py index 369db67..7e08c05 100644 --- a/scripts/test_domain_case_loop_step_state.py +++ b/scripts/test_domain_case_loop_step_state.py @@ -1617,6 +1617,46 @@ class DomainCaseLoopStepStateTests(unittest.TestCase): self.assertTrue(step_state["guarded_insufficiency_validated"]) self.assertEqual(step_state["acceptance_status"], "validated") + def test_guarded_mcp_discovery_debt_boundary_followup_validates(self) -> None: + answer_text = ( + "Да, доказать просрочку по компании ООО Альтернатива Плюс на 2020-12-31 нельзя, " + "потому что в подтвержденных данных не установлен срок оплаты.\n" + "Открытый долг на 35 472 380,36 руб. виден, но без due date это задолженность " + "без доказанной договорной просрочки." + ) + step_state = dcl.validate_step_contract( + { + "execution_status": "partial", + "reply_type": "partial_coverage", + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)срок.*оплат|due[- ]?date|договор", + "(?i)не установлен|не хватает|не подтвержд|не доказ", + "(?i)просроч", + "(?i)2020|2020-12-31|конец 2020", + ], + "fallback_type": "partial", + "truth_gate_contract_status": "partial_supported", + "truth_mode": "limited", + "answer_shape": "limited_with_reason", + "evidence_grade": "medium", + "mcp_discovery_response_applied": True, + "mcp_discovery_response_candidate_status": "ready_for_guarded_use", + "mcp_discovery_route_candidate_status": "ready_for_reviewed_execution", + "mcp_discovery_execution_handoff_can_use_guarded_response": True, + "assistant_text": answer_text, + "actual_direct_answer": ( + "Да, доказать просрочку по компании ООО Альтернатива Плюс на 2020-12-31 нельзя, " + "потому что в подтвержденных данных не установлен срок оплаты." + ), + "top_non_empty_lines": answer_text.splitlines(), + } + ) + + self.assertNotIn("required_answer_patterns_all_missing", step_state["violated_invariants"]) + self.assertTrue(step_state["guarded_insufficiency_validated"]) + self.assertEqual(step_state["acceptance_status"], "validated") + def test_expected_clarification_partial_answer_validates(self) -> None: answer_text = ( "Для рейтинга прибыльности номенклатуры нужен период.\n\n"