ARCH: проверить и расширить gate ответа MCP discovery

2026-04-20 12:23:53 +03:00
parent c744308223
commit 72804557aa
6 changed files with 254 additions and 4 deletions
@@ -0,0 +1,106 @@
+{
+  "schema_version": "domain_truth_harness_spec_v1",
+  "scenario_id": "address_truth_harness_phase19_mcp_discovery_response_gate",
+  "domain": "address_phase19_mcp_discovery_response_gate",
+  "title": "Phase 19 MCP discovery response gate replay",
+  "description": "Targeted AGENT replay for the first guarded MCP discovery answer-replacement gate. The scenario validates that exact supported routes remain authoritative, unsupported-but-understood counterparty lifecycle questions can use guarded discovered evidence, off-domain living chat is not hijacked, and internal MCP/runtime mechanics do not leak into the user-facing answer.",
+  "bindings": {},
+  "steps": [
+    {
+      "step_id": "step_01_human_smalltalk_sanity",
+      "title": "Human smalltalk remains living chat and does not expose discovery internals",
+      "question": "привет, ты на связи?",
+      "required_answer_patterns_any": [
+        "(?i)привет|на связи|готов|помочь"
+      ],
+      "forbidden_answer_patterns": [
+        "(?i)mcp",
+        "(?i)runtime_",
+        "(?i)query_documents",
+        "(?i)primitive"
+      ],
+      "criticality": "important",
+      "semantic_tags": [
+        "human_answer",
+        "mcp_discovery_gate_sanity"
+      ]
+    },
+    {
+      "step_id": "step_02_supported_counterparty_documents_stays_exact",
+      "title": "Supported counterparty documents route is not replaced by MCP discovery",
+      "question": "покажи документы по свк за 2020",
+      "allowed_reply_types": [
+        "factual",
+        "factual_with_explanation"
+      ],
+      "expected_intents": [
+        "list_documents_by_counterparty"
+      ],
+      "required_direct_answer_patterns_any": [
+        "(?i)свк|группа свк",
+        "(?i)документ|поступление|счет|счёт"
+      ],
+      "forbidden_direct_answer_patterns": [
+        "(?i)mcp",
+        "(?i)runtime_",
+        "(?i)query_documents",
+        "(?i)primitive",
+        "(?i)точный маршрут.*не подключ"
+      ],
+      "criticality": "critical",
+      "semantic_tags": [
+        "counterparty_documents",
+        "supported_route_not_hijacked_by_mcp_discovery"
+      ]
+    },
+    {
+      "step_id": "step_03_counterparty_lifecycle_uses_guarded_discovery",
+      "title": "Unsupported-but-understood counterparty lifecycle question uses guarded discovery answer",
+      "question": "сколько лет мы работаем с Группа СВК?",
+      "required_answer_patterns_all": [
+        "(?i)свк",
+        "(?i)1с|активност|подтвержд",
+        "(?i)вывод|оцен|инфер|можно оцен",
+        "(?i)юридическ|регистрац|не подтвержд|не доказ"
+      ],
+      "forbidden_answer_patterns": [
+        "(?i)точный маршрут.*не подключ",
+        "(?i)не буду подставлять",
+        "(?i)query_documents",
+        "(?i)query_movements",
+        "(?i)runtime_",
+        "(?i)planner_",
+        "(?i)catalog_",
+        "(?i)primitive"
+      ],
+      "criticality": "critical",
+      "semantic_tags": [
+        "mcp_discovery_response_gate",
+        "counterparty_lifecycle",
+        "unsupported_current_turn_meaning_boundary"
+      ]
+    },
+    {
+      "step_id": "step_04_off_domain_living_chat_not_hijacked",
+      "title": "Off-domain living chat remains human and is not hijacked by discovery carryover",
+      "question": "а чем капибара отличается от утки?",
+      "required_answer_patterns_any": [
+        "(?i)капибар.*утк|утк.*капибар",
+        "(?i)млекопита|птиц|грызун"
+      ],
+      "forbidden_answer_patterns": [
+        "(?i)свк",
+        "(?i)контрагент",
+        "(?i)mcp",
+        "(?i)query_documents",
+        "(?i)runtime_",
+        "(?i)primitive"
+      ],
+      "criticality": "critical",
+      "semantic_tags": [
+        "off_domain_living_chat",
+        "stale_replay_forbidden"
+      ]
+    }
+  ]
+}