АДРЕСНЫЙ РЕЖИМ -Step-5 - feat(assistant): стабилизация свободного LLM-роутинга, прическа маршрутов chat/address, прототип прогноза НДС
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# 2026-04-08_Address_Step5_UX_Smoke_v1
|
||||
|
||||
Generated at: 2026-04-08T13:53:34
|
||||
Questions file: X:\1C\NDC_1C\docs\ADDRESS\question_sets\step5_ux_smoke_2026-04-08.json
|
||||
Backend URL: http://127.0.0.1:8787/api/assistant/message
|
||||
LLM: local / qwen2.5-14b-instruct-1m @ http://127.0.0.1:1234
|
||||
Strict policy: route
|
||||
|
||||
## Totals
|
||||
- questions_total: 6
|
||||
- ok_200_count: 6
|
||||
- semantic_pass_count: 6
|
||||
- semantic_pass_rate: 1.0
|
||||
- route_pass_count: 6
|
||||
- route_pass_rate: 1.0
|
||||
- strict_pass_count: 6
|
||||
- strict_pass_rate: 1.0
|
||||
- factual_count: 6
|
||||
- partial_coverage_count: 0
|
||||
- clarification_required_count: 0
|
||||
- http_error_count: 0
|
||||
- llm_decomposition_attempted_count: 6
|
||||
- llm_decomposition_applied_count: 4
|
||||
- llm_fallback_count: 0
|
||||
- llm_fallback_rate: 0.0
|
||||
- tool_gate_blocked_count: 0
|
||||
- tool_gate_blocked_rate: 0.0
|
||||
- avg_elapsed_ms: 5562.2
|
||||
|
||||
## Files
|
||||
- run_summary.json
|
||||
- full_live_results.json
|
||||
- failures_only.json
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,10 @@
|
||||
# Response Audit: 2026-04-08_Address_Step5_UX_Smoke_v1
|
||||
|
||||
| id | strict | route_health | reply_type | intent | limited_reason | question | assistant_first_line |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| UX001 | True | ok_or_factual | factual | customer_revenue_and_payments | None | какие клиенты самые доходные | Топ-5 заказчиков по сумме поступлений: |
|
||||
| UX002 | True | ok_or_factual | factual | counterparty_population_and_roles | None | скока поставщиков в базе | Всего уникальных контрагентов в базе: 139. |
|
||||
| UX003 | True | ok_or_factual | factual | counterparty_activity_lifecycle | None | какие заказчики работали с нами в 2020 году | Активные заказчики в 2020 году: 13. |
|
||||
| UX004 | True | ok_or_factual | factual | contract_usage_overview | None | какие договоры давно не использовались | Использованных договоров: 291 из 394 (73.9%). |
|
||||
| UX005 | True | ok_or_factual | factual | list_documents_by_counterparty | None | покажи документы по нортону | Найдено документов по контрагенту: 12. |
|
||||
| UX006 | True | ok_or_factual | factual | bank_operations_by_contract | None | покажи операции по договору 19/15 | Найдено банковских операций по договору: 1. |
|
||||
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"run_id": "2026-04-08_Address_Step5_UX_Smoke_v1",
|
||||
"generated_at": "2026-04-08T13:53:34",
|
||||
"source_questions_file": "X:\\1C\\NDC_1C\\docs\\ADDRESS\\question_sets\\step5_ux_smoke_2026-04-08.json",
|
||||
"backend_url": "http://127.0.0.1:8787/api/assistant/message",
|
||||
"llm_provider": "local",
|
||||
"llm_model": "qwen2.5-14b-instruct-1m",
|
||||
"llm_base_url": "http://127.0.0.1:1234",
|
||||
"strict_policy": "route",
|
||||
"totals": {
|
||||
"questions_total": 6,
|
||||
"ok_200_count": 6,
|
||||
"semantic_pass_count": 6,
|
||||
"semantic_pass_rate": 1.0,
|
||||
"route_pass_count": 6,
|
||||
"route_pass_rate": 1.0,
|
||||
"strict_pass_count": 6,
|
||||
"strict_pass_rate": 1.0,
|
||||
"factual_count": 6,
|
||||
"partial_coverage_count": 0,
|
||||
"clarification_required_count": 0,
|
||||
"http_error_count": 0,
|
||||
"llm_decomposition_attempted_count": 6,
|
||||
"llm_decomposition_applied_count": 4,
|
||||
"llm_fallback_count": 0,
|
||||
"llm_fallback_rate": 0.0,
|
||||
"tool_gate_blocked_count": 0,
|
||||
"tool_gate_blocked_rate": 0.0,
|
||||
"avg_elapsed_ms": 5562.2
|
||||
},
|
||||
"distributions": {
|
||||
"reply_type": {
|
||||
"factual": 6
|
||||
},
|
||||
"actual_intent": {
|
||||
"customer_revenue_and_payments": 1,
|
||||
"counterparty_population_and_roles": 1,
|
||||
"counterparty_activity_lifecycle": 1,
|
||||
"contract_usage_overview": 1,
|
||||
"list_documents_by_counterparty": 1,
|
||||
"bank_operations_by_contract": 1
|
||||
},
|
||||
"actual_mode": {
|
||||
"address_query": 6
|
||||
},
|
||||
"mcp_call_status": {
|
||||
"matched_non_empty": 6
|
||||
},
|
||||
"limited_reason_category": {},
|
||||
"route_health": {
|
||||
"ok_or_factual": 6
|
||||
},
|
||||
"tool_gate_decision": {
|
||||
"run_address_lane": 6
|
||||
},
|
||||
"tool_gate_reason": {
|
||||
"address_mode_classifier_detected": 6
|
||||
}
|
||||
},
|
||||
"address_llm_predecompose_metrics": {
|
||||
"overall": {
|
||||
"llm_attempted": 6,
|
||||
"llm_applied": 4,
|
||||
"fallback_used": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"tool_gate_blocked": 0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"by_intent": {
|
||||
"customer_revenue_and_payments": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 0,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"counterparty_population_and_roles": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"counterparty_activity_lifecycle": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"contract_usage_overview": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 0,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"list_documents_by_counterparty": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"bank_operations_by_contract": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
# 2026-04-08_Address_Step5_UX_Smoke_v2
|
||||
|
||||
Generated at: 2026-04-08T13:55:47
|
||||
Questions file: X:\1C\NDC_1C\docs\ADDRESS\question_sets\step5_ux_smoke_2026-04-08.json
|
||||
Backend URL: http://127.0.0.1:8787/api/assistant/message
|
||||
LLM: local / qwen2.5-14b-instruct-1m @ http://127.0.0.1:1234
|
||||
Strict policy: route
|
||||
|
||||
## Totals
|
||||
- questions_total: 6
|
||||
- ok_200_count: 6
|
||||
- semantic_pass_count: 6
|
||||
- semantic_pass_rate: 1.0
|
||||
- route_pass_count: 6
|
||||
- route_pass_rate: 1.0
|
||||
- strict_pass_count: 6
|
||||
- strict_pass_rate: 1.0
|
||||
- factual_count: 6
|
||||
- partial_coverage_count: 0
|
||||
- clarification_required_count: 0
|
||||
- http_error_count: 0
|
||||
- llm_decomposition_attempted_count: 6
|
||||
- llm_decomposition_applied_count: 4
|
||||
- llm_fallback_count: 0
|
||||
- llm_fallback_rate: 0.0
|
||||
- tool_gate_blocked_count: 0
|
||||
- tool_gate_blocked_rate: 0.0
|
||||
- avg_elapsed_ms: 5551.5
|
||||
|
||||
## Files
|
||||
- run_summary.json
|
||||
- full_live_results.json
|
||||
- failures_only.json
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,10 @@
|
||||
# Response Audit: 2026-04-08_Address_Step5_UX_Smoke_v2
|
||||
|
||||
| id | strict | route_health | reply_type | intent | limited_reason | question | assistant_first_line |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| UX001 | True | ok_or_factual | factual | customer_revenue_and_payments | None | какие клиенты самые доходные | Топ-5 заказчиков по сумме поступлений: |
|
||||
| UX002 | True | ok_or_factual | factual | counterparty_population_and_roles | None | скока поставщиков в базе | Поставщиков (только supplier-роль): 79. |
|
||||
| UX003 | True | ok_or_factual | factual | counterparty_activity_lifecycle | None | какие заказчики работали с нами в 2020 году | Активные заказчики в 2020 году: 13. |
|
||||
| UX004 | True | ok_or_factual | factual | contract_usage_overview | None | какие договоры давно не использовались | Использованных договоров: 291 из 394 (73.9%). |
|
||||
| UX005 | True | ok_or_factual | factual | list_documents_by_counterparty | None | покажи документы по нортону | Найдено документов по контрагенту: 12. |
|
||||
| UX006 | True | ok_or_factual | factual | bank_operations_by_contract | None | покажи операции по договору 19/15 | Найдено банковских операций по договору: 1. |
|
||||
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"run_id": "2026-04-08_Address_Step5_UX_Smoke_v2",
|
||||
"generated_at": "2026-04-08T13:55:47",
|
||||
"source_questions_file": "X:\\1C\\NDC_1C\\docs\\ADDRESS\\question_sets\\step5_ux_smoke_2026-04-08.json",
|
||||
"backend_url": "http://127.0.0.1:8787/api/assistant/message",
|
||||
"llm_provider": "local",
|
||||
"llm_model": "qwen2.5-14b-instruct-1m",
|
||||
"llm_base_url": "http://127.0.0.1:1234",
|
||||
"strict_policy": "route",
|
||||
"totals": {
|
||||
"questions_total": 6,
|
||||
"ok_200_count": 6,
|
||||
"semantic_pass_count": 6,
|
||||
"semantic_pass_rate": 1.0,
|
||||
"route_pass_count": 6,
|
||||
"route_pass_rate": 1.0,
|
||||
"strict_pass_count": 6,
|
||||
"strict_pass_rate": 1.0,
|
||||
"factual_count": 6,
|
||||
"partial_coverage_count": 0,
|
||||
"clarification_required_count": 0,
|
||||
"http_error_count": 0,
|
||||
"llm_decomposition_attempted_count": 6,
|
||||
"llm_decomposition_applied_count": 4,
|
||||
"llm_fallback_count": 0,
|
||||
"llm_fallback_rate": 0.0,
|
||||
"tool_gate_blocked_count": 0,
|
||||
"tool_gate_blocked_rate": 0.0,
|
||||
"avg_elapsed_ms": 5551.5
|
||||
},
|
||||
"distributions": {
|
||||
"reply_type": {
|
||||
"factual": 6
|
||||
},
|
||||
"actual_intent": {
|
||||
"customer_revenue_and_payments": 1,
|
||||
"counterparty_population_and_roles": 1,
|
||||
"counterparty_activity_lifecycle": 1,
|
||||
"contract_usage_overview": 1,
|
||||
"list_documents_by_counterparty": 1,
|
||||
"bank_operations_by_contract": 1
|
||||
},
|
||||
"actual_mode": {
|
||||
"address_query": 6
|
||||
},
|
||||
"mcp_call_status": {
|
||||
"matched_non_empty": 6
|
||||
},
|
||||
"limited_reason_category": {},
|
||||
"route_health": {
|
||||
"ok_or_factual": 6
|
||||
},
|
||||
"tool_gate_decision": {
|
||||
"run_address_lane": 6
|
||||
},
|
||||
"tool_gate_reason": {
|
||||
"address_mode_classifier_detected": 6
|
||||
}
|
||||
},
|
||||
"address_llm_predecompose_metrics": {
|
||||
"overall": {
|
||||
"llm_attempted": 6,
|
||||
"llm_applied": 4,
|
||||
"fallback_used": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"tool_gate_blocked": 0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"by_intent": {
|
||||
"customer_revenue_and_payments": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 0,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"counterparty_population_and_roles": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"counterparty_activity_lifecycle": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"contract_usage_overview": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 0,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"list_documents_by_counterparty": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
},
|
||||
"bank_operations_by_contract": {
|
||||
"total": 1,
|
||||
"llm_attempted": 1,
|
||||
"llm_applied": 1,
|
||||
"fallback_used": 0,
|
||||
"tool_gate_blocked": 0,
|
||||
"fallback_rate": 0.0,
|
||||
"gate_block_rate": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user