Initial import NDC_1C
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-001",
|
||||
"raw_question": "По каким поставщикам на конец июня не бьются взаиморасчеты, покажи документы, оплаты и хвосты.",
|
||||
"expected": {
|
||||
"intent_class": "cross_entity",
|
||||
"route_hint": "hybrid_store_plus_live",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"accounts_mentioned": ["60"],
|
||||
"expected_output_shape": "reconciliation_report"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.",
|
||||
"expected": {
|
||||
"intent_class": "heavy_analytical",
|
||||
"route_hint": "batch_refresh_then_store",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": [],
|
||||
"expected_output_shape": "ranked_list"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-003",
|
||||
"raw_question": "Покажи документ по номеру 000123 и строку проводки, нужен точный source-of-record.",
|
||||
"expected": {
|
||||
"intent_class": "drilldown_explain",
|
||||
"route_hint": "live_mcp_drilldown",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": [],
|
||||
"expected_output_shape": "evidence_chain"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-004",
|
||||
"raw_question": "По 97 счету проверь, где возможна ошибка дат начала и окончания списания.",
|
||||
"expected": {
|
||||
"intent_class": "rule_based_account_control",
|
||||
"route_hint": "store_feature_risk",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": ["97"],
|
||||
"expected_output_shape": "prioritized_review_list"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-005",
|
||||
"raw_question": "Есть ли аномальные материалы на счете 10, которые зависли и выглядят нелогично?",
|
||||
"expected": {
|
||||
"intent_class": "anomaly_probe",
|
||||
"route_hint": "store_feature_risk",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": ["10"],
|
||||
"expected_output_shape": "anomaly_summary"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-006",
|
||||
"raw_question": "По каким реализациям 90/62 хвосты не закрылись оплатой, разложи по цепочке документов.",
|
||||
"expected": {
|
||||
"intent_class": "cross_entity",
|
||||
"route_hint": "hybrid_store_plus_live",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"accounts_mentioned": ["90", "62"],
|
||||
"expected_output_shape": "reconciliation_report"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?",
|
||||
"expected": {
|
||||
"intent_class": "heavy_analytical",
|
||||
"route_hint": "batch_refresh_then_store",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": [],
|
||||
"expected_output_shape": "prioritized_review_list"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"expected": {
|
||||
"intent_class": "drilldown_explain",
|
||||
"route_hint": "live_mcp_drilldown",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": ["51"],
|
||||
"expected_output_shape": "evidence_chain"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-009",
|
||||
"raw_question": "Где у нас пахнет ручной ошибкой по июню?",
|
||||
"expected": {
|
||||
"intent_class": "ambiguous_human_query",
|
||||
"route_hint": "batch_refresh_then_store",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": [],
|
||||
"expected_output_shape": "anomaly_summary"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"case_id": "NQ-010",
|
||||
"raw_question": "Какое сальдо по 68.02 за июнь?",
|
||||
"expected": {
|
||||
"intent_class": "simple_factual",
|
||||
"route_hint": "store_canonical",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"accounts_mentioned": ["68.02"],
|
||||
"expected_output_shape": "point_answer"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"case_id": "NQ-1774273974528",
|
||||
"raw_question": "По каким поставщикам не бьются взаиморасчеты по 60 счету, и разложи это по документам, оплатам и закрывающим?",
|
||||
"expected": {
|
||||
"intent_class": "cross_entity",
|
||||
"route_hint": "hybrid_store_plus_live",
|
||||
"requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"accounts_mentioned": [
|
||||
"60"
|
||||
],
|
||||
"expected_output_shape": "reconciliation_report"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval--JtoGUT4Aw",
|
||||
"timestamp": "2026-03-24T10:06:13.697Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "4Psn1fS8nRyNqU",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "THlkt4T0wpen4O",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "0d60S_Wt4316XD",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-1ZO6LtR6Re",
|
||||
"timestamp": "2026-03-25T19:37:38.656Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "an-nihAYeEc7ej",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "bfoYYavqkvTuyJ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "nI7bKto-8bnMbk",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-4xS4Pyr7Uv",
|
||||
"timestamp": "2026-03-25T20:06:37.364Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "wqgfbCw0SkZgCM",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "g-Ocfo0aK3GK59",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "OzQNCkSVXVQtGh",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-68KSDPHUoo",
|
||||
"timestamp": "2026-03-25T19:04:30.554Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "6arBKiTtUytVUF",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "dyHwvmvxb_-laX",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "wpmj4lEbYxntfj",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-73H4_ECGyy",
|
||||
"timestamp": "2026-03-24T10:05:43.126Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "HqCLdEdu0pq2wL",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "C53x-SO0IAGniS",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "hfI0mOdPmNExaq",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"run_id": "eval-7MQmPgo3_O",
|
||||
"timestamp": "2026-03-23T18:38:44.797Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_1",
|
||||
"schema_version": "v2_0_1",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "GjMJwja-ODnkHV",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "2E-aeau6g5EmA9",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "yP_hHlfC_fEVUr",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-CsaZeqddTt",
|
||||
"timestamp": "2026-03-25T18:04:12.536Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "_OL1Xt8G_uK5Wd",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "ATj7VbuQWryGMO",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "MN4pN5swyPjwyF",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-DZfBsZawGS",
|
||||
"timestamp": "2026-03-25T20:09:33.541Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "97U78ZwtYeEZFS",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "kKyl7khxokZPMW",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"run_id": "eval-DuRnhPquJ1",
|
||||
"timestamp": "2026-03-23T18:40:07.215Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_1",
|
||||
"schema_version": "v2_0_1",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "EUVfhx5Lnr6fxW",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "traFZsOhAjkdGA",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "kkUNPiK_1ZbhGK",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-EJ5CZqfWe3",
|
||||
"timestamp": "2026-03-25T18:05:43.704Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "7tKYIxvBa1zYc8",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "QurXir1k7Fr_0d",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "5iFOAPUVdV5tcM",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-GLNMT_DXlm",
|
||||
"timestamp": "2026-03-24T09:27:08.489Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "dRvzM7klA7Gjcs",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "DZVob3OxZZsX9n",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "6Ln1gw3a-oL2Hg",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"run_id": "eval-HQ5HtxrQpc",
|
||||
"timestamp": "2026-03-23T18:38:22.095Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2",
|
||||
"schema_version": "v2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 33.33,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 0,
|
||||
"soft_assumption_used_fragment_rate": 0,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "wlDq7CMwRfHOT6",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "1YBeG1qLyd-owb",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "hPIaIVlLuLdyus",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"run_id": "eval-IpucvMroko",
|
||||
"timestamp": "2026-03-23T17:59:01.977Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2",
|
||||
"schema_version": "v2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 33.33,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"clarification": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "lzwdL3W9zRCLVe",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"trace_id": "b2LgkaTILQ8a4G",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"trace_id": "DYWHGwbQ2K4p1K",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-Jj5FenxK1M",
|
||||
"timestamp": "2026-03-25T20:49:22.808Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Iybc1qMcMhKCwg",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "IqBJez1iuJV2Eh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "sYU5vPDwtM1UZr",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-KoIThx--qy",
|
||||
"timestamp": "2026-03-25T20:09:40.107Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "sqQQMHUAOa_6q7",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "CoJBGKfeqqoZos",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
{
|
||||
"run_id": "eval-LyUcB42oK9",
|
||||
"timestamp": "2026-03-23T15:21:58.788Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v1_1_1",
|
||||
"dataset": {
|
||||
"source": "file",
|
||||
"file": "normalizer_eval_v1_1_30cases.json"
|
||||
},
|
||||
"cases_total": 5,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 40,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"baseline_metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"baseline_delta": {
|
||||
"schema_validation_pass_rate": 0,
|
||||
"intent_class_accuracy": -32.73,
|
||||
"route_hint_accuracy": 9.09,
|
||||
"causal_flag_accuracy": 18.18,
|
||||
"high_confidence_error_rate": -9.09
|
||||
},
|
||||
"class_accuracy": {
|
||||
"drilldown_explain": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"period_close_risk": {
|
||||
"total": 2,
|
||||
"passed": 0,
|
||||
"accuracy_percent": 0
|
||||
},
|
||||
"anomaly_probe": {
|
||||
"total": 1,
|
||||
"passed": 0,
|
||||
"accuracy_percent": 0
|
||||
}
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0,
|
||||
"guidance": {
|
||||
"forensic_calls_max": 10,
|
||||
"final_eval_calls_max": 30,
|
||||
"target_total_calls_max": 40,
|
||||
"hard_cap_calls_max": 45
|
||||
}
|
||||
},
|
||||
"mismatches": [
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||||
"trace_id": "LyziNYqDRnmIav"
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||||
"trace_id": "mo09Q4j0hOVWgw"
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||||
"trace_id": "zLXsDrsg2vR5Qs"
|
||||
}
|
||||
],
|
||||
"bad_confidence_cases": [],
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "rnUogTOSYvvR0g",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-005",
|
||||
"raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "LSKGq9-z1wUR10",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "LyziNYqDRnmIav",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "mo09Q4j0hOVWgw",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "zLXsDrsg2vR5Qs",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-M5jMc2B9tS",
|
||||
"timestamp": "2026-03-24T08:24:25.436Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "5ocqhHq3Y62OGK",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "vYMdsPSQ4lngmx",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "u67X39gO0Sltw5",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-Np5p0yPVr4",
|
||||
"timestamp": "2026-03-24T11:00:15.280Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "4rEksqx_Ui7UaF",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "tlE4Gc3MWF-QUH",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "6Abphg2DtdAQIn",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-PBFcuAprq9",
|
||||
"timestamp": "2026-03-25T20:41:15.150Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "z_EYXsYap-7QNj",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "pN0dSjal50OBYy",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "hfWYp5j8ziWsjx",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
{
|
||||
"run_id": "eval-Rgb_3vYT7U",
|
||||
"timestamp": "2026-03-23T15:23:22.517Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v1_1_1",
|
||||
"dataset": {
|
||||
"source": "file",
|
||||
"file": "normalizer_eval_v1_1_30cases.json"
|
||||
},
|
||||
"cases_total": 5,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 80,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"baseline_metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"baseline_delta": {
|
||||
"schema_validation_pass_rate": 0,
|
||||
"intent_class_accuracy": 7.27,
|
||||
"route_hint_accuracy": 9.09,
|
||||
"causal_flag_accuracy": 18.18,
|
||||
"high_confidence_error_rate": -9.09
|
||||
},
|
||||
"class_accuracy": {
|
||||
"drilldown_explain": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"period_close_risk": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"anomaly_probe": {
|
||||
"total": 1,
|
||||
"passed": 0,
|
||||
"accuracy_percent": 0
|
||||
}
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0,
|
||||
"guidance": {
|
||||
"forensic_calls_max": 10,
|
||||
"final_eval_calls_max": 30,
|
||||
"target_total_calls_max": 40,
|
||||
"hard_cap_calls_max": 45
|
||||
}
|
||||
},
|
||||
"mismatches": [
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||||
"trace_id": "ExCllsbZwIgrUH"
|
||||
}
|
||||
],
|
||||
"bad_confidence_cases": [],
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "XPFrVhyjSsyiPO",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-005",
|
||||
"raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "-kXMi9OZ_e0-jx",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "M0jolGBzOoJoa3",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "ExCllsbZwIgrUH",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "uDmYjA0uMNJKZC",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-ShSTDlSKMR",
|
||||
"timestamp": "2026-03-25T20:41:15.680Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "1nRoFDbAGJmouM",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "jjy4xNpnrCF-jR",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-Vgie6SCY1Z",
|
||||
"timestamp": "2026-03-24T10:18:46.934Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "2Iv06y6I4tdoa_",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Mx14pWPKBmvEDx",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "mWZV1v9J6SfNlH",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-WezQxQD1Jb",
|
||||
"timestamp": "2026-03-24T11:52:42.608Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "U2mjAb-6fBtEzs",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "fjGPG5W1lXz_PU",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "nz-WOygg2dppTf",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"run_id": "eval-X_CQK9BieE",
|
||||
"timestamp": "2026-03-23T18:37:36.909Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2",
|
||||
"schema_version": "v2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 33.33,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 0,
|
||||
"soft_assumption_used_fragment_rate": 0,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "frzs0a7NGWGkGh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "A24YOwbPezcaWE",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "jK2gs4w1KD84Ca",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-YXNecDV0dP",
|
||||
"timestamp": "2026-03-25T20:41:09.151Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "SOXSWk2ulnVptZ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "hmjJx04ZfV0JN1",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"run_id": "eval-YxrhL2dCcH",
|
||||
"timestamp": "2026-03-23T13:55:23.242Z",
|
||||
"cases_total": 11,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-001",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "WBCFIgEMf1OFC-"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "COIYD-5V_3EChj"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-003",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "8s-4eCbeXywVVK"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-004",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": false,
|
||||
"trace_id": "h5BLdC1oBO0wY6"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-005",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "p7A-wb3EfMen5P"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-006",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "Um8MUe8yfaKLxw"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "I6V3tELpvi6k1i"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": false,
|
||||
"trace_id": "iel5ScdccVZ4zT"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-009",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": false,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "1C9MATbKvo5FnF"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-010",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "ti9EWEi85p-RnD"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-1774273974528",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "lSYWnMUueaZgc2"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"run_id": "eval-Z9wlxFh4Wa",
|
||||
"timestamp": "2026-03-23T15:49:49.563Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v1_1_2",
|
||||
"dataset": {
|
||||
"source": "file",
|
||||
"file": "normalizer_eval_v1_1_30cases.json"
|
||||
},
|
||||
"cases_total": 5,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 100,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"baseline_metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"baseline_delta": {
|
||||
"schema_validation_pass_rate": 0,
|
||||
"intent_class_accuracy": 27.27,
|
||||
"route_hint_accuracy": 9.09,
|
||||
"causal_flag_accuracy": 18.18,
|
||||
"high_confidence_error_rate": -9.09
|
||||
},
|
||||
"class_accuracy": {
|
||||
"heavy_analytical": {
|
||||
"total": 3,
|
||||
"passed": 3,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"period_close_risk": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
}
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0,
|
||||
"guidance": {
|
||||
"forensic_calls_max": 10,
|
||||
"final_eval_calls_max": 30,
|
||||
"target_total_calls_max": 40,
|
||||
"hard_cap_calls_max": 45
|
||||
}
|
||||
},
|
||||
"mismatches": [],
|
||||
"bad_confidence_cases": [],
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "VsAOz81dbRjpxp",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "i6vYaeziX0JPGh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-HA-004",
|
||||
"raw_question": "Дай обзорный риск-срез перед сдачей отчетности: где максимальная концентрация ошибок.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "JU-GLRW5-TVJYu",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "EJimLLNg_2cB-D",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "2d9EQnCBaYkcsB",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-ZtTBKDJHAs",
|
||||
"timestamp": "2026-03-25T19:02:19.044Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "pT6gram76cCQOZ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "KQY-307LTx_FuD",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "UjGTNB09p9JzPJ",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-_pdPEhkMOU",
|
||||
"timestamp": "2026-03-25T20:07:36.377Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "TGll0b_TnbyKmx",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "ac1fvpXRiEtQza",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"run_id": "eval-aOEoelkQb2",
|
||||
"timestamp": "2026-03-23T18:09:06.293Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": false,
|
||||
"prompt_version": "normalizer_v2",
|
||||
"schema_version": "v2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 10
|
||||
},
|
||||
"cases_total": 10,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 80,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 11,
|
||||
"retries_used": 1
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 4,
|
||||
"store_feature_risk": 6
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2,
|
||||
"clarification": 8
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Какие реализации на конец периода выглядят так, будто они зависли и будут портить картину по выручке, если их не проверить заранее?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"trace_id": "8_BzV102qV2iTO",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи реализации, где хвост выглядит особенно неприятно: сумма не маленькая, возраст хвоста уже заметный, и при этом не видно нормального завершения цепочки.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "3rUQkDkh-JlGl6",
|
||||
"request_count_for_case": 2
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "По каким продажам на конец месяца видно, что бухгалтер потом будет долго распутывать, почему все это не сошлось нормально?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"trace_id": "SCnaV6ceFjE2IH",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-004",
|
||||
"raw_question": "Где банк и бухгалтерский контур, скорее всего, расходятся не по одной строке, а по паттерну, который уже начинает повторяться?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "n33S9rLjDQvvRh",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-005",
|
||||
"raw_question": "Есть ли такие зависшие авансы, которые уже давно надо было либо закрыть, либо хотя бы перепроверить руками?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "mevf8cflx6G80P",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-006",
|
||||
"raw_question": "Что сейчас лежит на 10 счете так, будто это уже давно надо было либо списать, либо хотя бы проверить, почему оно до сих пор висит?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "fwB6C_fBfZqdzu",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-007",
|
||||
"raw_question": "Покажи позиции по материалам, где возможен эффект \"вроде сумма не огромная, но учетная логика выглядит криво\".",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "medium",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "6BAiubmfHx17q_",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-008",
|
||||
"raw_question": "Есть ли такие расходы будущих периодов, которые заведены, но по ним не видно нормальной ежемесячной жизни, как будто запись повисла сама по себе?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "1KBRovASAt6sTv",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-009",
|
||||
"raw_question": "Покажи объекты ОС, где риск не в сумме, а в том, что карточка и логика начисления выглядят подозрительно и могут аукнуться позже.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "pkExx_p7wcGIMk",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-010",
|
||||
"raw_question": "Есть ли движения по счету 51, которые выглядят корректно по сумме, но по смыслу оставляют после себя подозрительный хвост?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"trace_id": "BI7ljrWDvTgbKg",
|
||||
"request_count_for_case": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-ajxfqukgjf",
|
||||
"timestamp": "2026-03-24T08:34:12.228Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "L_8LImaABypEPe",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Inn7bcOXXKXF8X",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "TIT0qqVZod729c",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-alKADONJvL",
|
||||
"timestamp": "2026-03-24T11:03:55.005Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "YtC2yvnVwSTopk",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "lMGCbyGIqa5ntM",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "v-AM3y6DR8BKP6",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
{
|
||||
"run_id": "eval-baY1nPi1rI",
|
||||
"timestamp": "2026-03-23T18:59:54.829Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": false,
|
||||
"prompt_version": "normalizer_v2_0_1",
|
||||
"schema_version": "v2_0_1",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 26
|
||||
},
|
||||
"cases_total": 26,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 96.15,
|
||||
"scope_in_scope_rate": 92.31,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 15.38,
|
||||
"avg_fragments_per_message": 1.12,
|
||||
"out_of_scope_fragment_rate": 3.45,
|
||||
"routed_fragment_rate": 79.31,
|
||||
"no_route_fragment_rate": 20.69,
|
||||
"executable_with_soft_assumptions_rate": 75,
|
||||
"soft_assumption_used_fragment_rate": 75,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 27,
|
||||
"retries_used": 1
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 11,
|
||||
"store_feature_risk": 7,
|
||||
"batch_refresh_then_store": 4,
|
||||
"no_route": 6,
|
||||
"store_canonical": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 21,
|
||||
"clarification": 4,
|
||||
"out_of_scope": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Сделай общий предзакрывающий срез по июню: где у нас самые опасные узлы, которые могут дать искажение по выручке, взаиморасчётам и остаткам одновременно, и разложи это по приоритету ручной проверки.",
|
||||
"validation_passed": false,
|
||||
"message_in_scope": null,
|
||||
"scope_confidence": null,
|
||||
"contains_multiple_tasks": null,
|
||||
"fragments_total": 0,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "SatgafwxwDR9BU",
|
||||
"request_count_for_case": 2
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи контрагентов, по которым хвосты уже выглядят не как обычная операционная задержка, а как системная проблема в связке документы → оплаты → закрывающие.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 2,
|
||||
"trace_id": "n1MTEG5Q29gcYT",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Есть ли у нас такие продажи, где формально всё проведено, но по логике движения денег, документов и взаиморасчётов видно, что цепочка собрана криво и в любой момент может всплыть.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "fUhDiDLbuJiG6-",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-004",
|
||||
"raw_question": "Собери список самых неприятных кейсов по 90/62 на конец месяца: чтобы было видно сумму, возраст хвоста, признаки кривой связки и вероятность, что это не просто неоплата, а проблема учёта.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "1rPOulMwiTUNhk",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-005",
|
||||
"raw_question": "Где по банку есть ощущение, что выписка, документ и проводка живут отдельно друг от друга, и уже начинает формироваться повторяющийся паттерн, а не единичный косяк.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 2,
|
||||
"trace_id": "kaY792kzieMaGz",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-006",
|
||||
"raw_question": "Покажи банковские движения, которые выглядят прилично по сумме, но после них остаётся бухгалтерский хвост, который не объясняется нормальной хозяйственной логикой.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "ZWbQvRRpw4S66P",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-007",
|
||||
"raw_question": "Есть ли такие участки по 51 счёту, где проблема, скорее всего, не в платеже, а в том, что закрытие операции ушло не тем документом или не в тот смысловой узел.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "7_Fn4g4trZlueK",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-008",
|
||||
"raw_question": "Покажи товарные позиции, по которым уже можно подозревать, что реализация, приход и остаток между собой не держат нормальную учётную конструкцию.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "tUhkJVKXEc3rI-",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-009",
|
||||
"raw_question": "Где по складу и реализации видно, что проблема не просто в количестве, а в том, что себестоимость или подтверждение движения опираются на слабую или разваленную цепочку.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Ha3VJR9Cfgu7k_",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-010",
|
||||
"raw_question": "Есть ли материалы на 10 счёте, которые зависли так, что это уже выглядит не как забытая мелочь, а как системная дыра в логике участка.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "8fdkn-EtZynWPb",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-011",
|
||||
"raw_question": "Покажи позиции по материалам, где сумма сама по себе не пугает, но характер остатка и движения намекает, что потом это может аукнуться при разборе периода.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "HNAXW_JV6E_hp-",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-012",
|
||||
"raw_question": "Какие записи на 97 счёте больше всего похожи на историю, где документ завели, а дальше нормальной жизни у него не случилось: срок, списание и поведение записи друг другу противоречат.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "syUHcCf_EoioxW",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-013",
|
||||
"raw_question": "Есть ли по РБП такие кейсы, где ошибка не в одной дате, а в том, что сама логика начала, окончания и длительности выглядит бухгалтерски сомнительно.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "9JP7hooywVr3D1",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-014",
|
||||
"raw_question": "Покажи объекты ОС, где карточка, срок и логика начисления амортизации выглядят так, будто объект поставили на учёт скорее “как-нибудь”, чем по нормальной модели.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "0a5H84MIV-WDv4",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-015",
|
||||
"raw_question": "Собери список самых подозрительных объектов ОС: не по сумме, а по риску того, что потом придётся долго объяснять, почему карточка и начисления не бьются между собой.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "mIR3Cru_dmM5PQ",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-016",
|
||||
"raw_question": "Где у нас в целом по июню повторяется один и тот же тип проблемной связи, а не просто всплывают разовые косяки по разным участкам.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "V6VRAtxXFVlx02",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-017",
|
||||
"raw_question": "Если смотреть на компанию как на единый контур, какие три зоны сейчас опаснее всего с точки зрения накопленного технического мусора в учёте, который может вылезти на закрытии.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "vuvha8oq0Y67kU",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-018",
|
||||
"raw_question": "Разложи по приоритету не просто ошибки, а именно те места, где человеку потом придётся дольше всего вручную распутывать, что произошло и почему это не сошлось.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "dtTCTe9sMiGv2F",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-019",
|
||||
"raw_question": "Есть ли такие висящие истории, где по верхнему уровню всё выглядит терпимо, но если копнуть вглубь, видно, что несколько сущностей держатся на слабой или ложной связи.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "avKa64EwOxGaLl",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-020",
|
||||
"raw_question": "Покажи мне не просто аномалии, а аномалии второго порядка: такие места, где отдельная ошибка не страшна, но в сочетании с другим участком уже начинает ломать картину периода.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "2VO4fwiW6_quNT",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-021",
|
||||
"raw_question": "Что сейчас в июне выглядит как история, которую бухгалтер отложил бы “на потом”, но именно из-за этого она может стать самой неприятной на закрытии.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "37Yw34xBrAk8B4",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-022",
|
||||
"raw_question": "Если бы нужно было за один заход проверить только пять узлов учёта перед закрытием, какие пять ты бы выбрал и по каким объективным признакам.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "7w49YyxNr1wJs1",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-023",
|
||||
"raw_question": "Есть ли кейсы, где один и тот же контрагент или объект всплывает сразу в нескольких подозрительных связях, и это уже похоже на концентратор проблем, а не на случайность.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 2,
|
||||
"trace_id": "J9Yi0e6BjngR3a",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-024",
|
||||
"raw_question": "Покажи вопросы, которые можно решить только через причинную раскладку по документам и проводкам, а не просто через общий обзор подозрительных мест.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "wKyodOpouHsh1D",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-025",
|
||||
"raw_question": "Есть ли в нашей базе куски, которые выглядят так, будто их кто-то руками “дособирал”, и теперь формально всё стоит, но смысловая логика учёта у этого шатается.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "WTXGtMV3WRWhbP",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-026",
|
||||
"raw_question": "Как в целом правильно отражать расходы будущих периодов по лицензиям и подпискам по российским правилам, если говорить вообще, а не по нашей базе.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "dtXqHsYutlsp6q",
|
||||
"request_count_for_case": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-dlId-aZfVF",
|
||||
"timestamp": "2026-03-25T20:07:35.884Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "BDdPd1cby0Naq5",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "0MPqfa0EDjwWI4",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "iEb2JMYq0w8RtQ",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-dp0byorFPG",
|
||||
"timestamp": "2026-03-25T20:49:23.341Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "5kzKcWpJnu7_3B",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "NS4YJbRNVBFcKp",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-dwCPZM_ejT",
|
||||
"timestamp": "2026-03-25T17:46:08.300Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "UUlQqFfk26VQov",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "RJFvgx3uy0Q1JS",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "7wQRtY1JdwYxNt",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-evxc50N67z",
|
||||
"timestamp": "2026-03-24T11:02:28.296Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "xVicKw7FNogsTY",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "uEexVMbap7HEmc",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "SHvj9UZGmr_91Q",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"run_id": "eval-f3Emsynsd7",
|
||||
"timestamp": "2026-03-23T18:39:08.681Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_1",
|
||||
"schema_version": "v2_0_1",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "XAsDRTakWi0P30",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Nd7NIhk9esvqei",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "f8z3blshrmwasO",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-iODkIA42hi",
|
||||
"timestamp": "2026-03-25T20:07:25.149Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "zo9JrPf3beUQkG",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "xGQ-UmA8CgVFB0",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-kXBMJEsAle",
|
||||
"timestamp": "2026-03-25T17:24:03.560Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "_pjZQDyPCMXY2S",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "ghOyUfJmLElHAZ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Lobw1JTAN-BCt0",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-nLSBmizWaN",
|
||||
"timestamp": "2026-03-24T11:05:48.660Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "WW_6cPIHnnM_B7",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "oI0HQEkSJeeGq9",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "4mwEApgPn1GRKP",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-o8-zix4b7i",
|
||||
"timestamp": "2026-03-24T10:03:34.246Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "tldUd3gXaur1GT",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "USjOvRNZSWqCKo",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "LNKIRO9LV-Fgen",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-p-M8D171jK",
|
||||
"timestamp": "2026-03-25T17:45:28.132Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "5tIPz1DSKs5d8l",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "JcO3P0g-usjQou",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "ULGl6uGRjFgcL5",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-pi-L1EyJmP",
|
||||
"timestamp": "2026-03-25T20:09:39.601Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "mVq2cmfoUEgihE",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "qev2BWo8Na1rLK",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "A1jbqz59WIG0Zg",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-qWEMr5wpHb",
|
||||
"timestamp": "2026-03-25T18:03:32.256Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "hQa_n0ZeGn4CdB",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "36NoS_DQb2dEU8",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "QOFWG6bYav-u9X",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,209 @@
|
||||
{
|
||||
"run_id": "eval-rcY2eNdWNz",
|
||||
"timestamp": "2026-03-23T15:23:51.992Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v1_1_1",
|
||||
"dataset": {
|
||||
"source": "file",
|
||||
"file": "normalizer_eval_v1_1_30cases.json"
|
||||
},
|
||||
"cases_total": 5,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 100,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"baseline_metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"baseline_delta": {
|
||||
"schema_validation_pass_rate": 0,
|
||||
"intent_class_accuracy": 27.27,
|
||||
"route_hint_accuracy": 9.09,
|
||||
"causal_flag_accuracy": 18.18,
|
||||
"high_confidence_error_rate": -9.09
|
||||
},
|
||||
"class_accuracy": {
|
||||
"drilldown_explain": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"period_close_risk": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"anomaly_probe": {
|
||||
"total": 1,
|
||||
"passed": 1,
|
||||
"accuracy_percent": 100
|
||||
}
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0,
|
||||
"guidance": {
|
||||
"forensic_calls_max": 10,
|
||||
"final_eval_calls_max": 30,
|
||||
"target_total_calls_max": 40,
|
||||
"hard_cap_calls_max": 45
|
||||
}
|
||||
},
|
||||
"mismatches": [],
|
||||
"bad_confidence_cases": [],
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "fjIuLbOgy1GlvC",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-005",
|
||||
"raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "hYPahELBOHx8Gp",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "sMardbub40y2av",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "anomaly_probe",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "Vj-YEh8keRj_lA",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "low",
|
||||
"trace_id": "ziCohC1zzdxZuq",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-sP2C9OyC53",
|
||||
"timestamp": "2026-03-24T08:24:57.416Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "VsSFEiq632vYji",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Wy6bXNRlsVh4ha",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "LFZVrcgKHxiajr",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-t7-IMyNQxX",
|
||||
"timestamp": "2026-03-25T20:06:37.481Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "EVg7hkz9frxBoB",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "jfu04LkJQ6gWJD",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,646 @@
|
||||
{
|
||||
"run_id": "eval-uQiZ1L_ga6",
|
||||
"timestamp": "2026-03-24T08:40:17.177Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": false,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 26
|
||||
},
|
||||
"cases_total": 26,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 96.15,
|
||||
"multi_intent_detected_rate": 3.85,
|
||||
"clarification_required_rate": 15.38,
|
||||
"avg_fragments_per_message": 1.15,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 80,
|
||||
"no_route_fragment_rate": 20,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 26.67,
|
||||
"soft_assumption_used_fragment_rate": 26.67,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 26,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 30,
|
||||
"checks_passed": 30
|
||||
},
|
||||
"route_distribution": {
|
||||
"batch_refresh_then_store": 3,
|
||||
"no_route": 6,
|
||||
"hybrid_store_plus_live": 16,
|
||||
"store_feature_risk": 1,
|
||||
"store_canonical": 4
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"partial": 2,
|
||||
"none": 19,
|
||||
"clarification": 4,
|
||||
"out_of_scope": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Сделай общий предзакрывающий срез по июню: где у нас самые опасные узлы, которые могут дать искажение по выручке, взаиморасчётам и остаткам одновременно, и разложи это по приоритету ручной проверки.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": true,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "partial",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "xVb4xrCqvnOovk",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи контрагентов, по которым хвосты уже выглядят не как обычная операционная задержка, а как системная проблема в связке документы → оплаты → закрывающие.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "UBTjVJbclcrCeq",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Есть ли у нас такие продажи, где формально всё проведено, но по логике движения денег, документов и взаиморасчётов видно, что цепочка собрана криво и в любой момент может всплыть.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "gD3HqUuSUpZ5Tp",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-004",
|
||||
"raw_question": "Собери список самых неприятных кейсов по 90/62 на конец месяца: чтобы было видно сумму, возраст хвоста, признаки кривой связки и вероятность, что это не просто неоплата, а проблема учёта.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "HAKmX8tLb058r_",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-005",
|
||||
"raw_question": "Где по банку есть ощущение, что выписка, документ и проводка живут отдельно друг от друга, и уже начинает формироваться повторяющийся паттерн, а не единичный косяк.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 2,
|
||||
"trace_id": "ihEgIB61dV4bab",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-006",
|
||||
"raw_question": "Покажи банковские движения, которые выглядят прилично по сумме, но после них остаётся бухгалтерский хвост, который не объясняется нормальной хозяйственной логикой.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Er-6mY_qCQh0us",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-007",
|
||||
"raw_question": "Есть ли такие участки по 51 счёту, где проблема, скорее всего, не в платеже, а в том, что закрытие операции ушло не тем документом или не в тот смысловой узел.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "VOnX5GsSrOl4ed",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-008",
|
||||
"raw_question": "Покажи товарные позиции, по которым уже можно подозревать, что реализация, приход и остаток между собой не держат нормальную учётную конструкцию.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "JPLZR1frSolzGf",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-009",
|
||||
"raw_question": "Где по складу и реализации видно, что проблема не просто в количестве, а в том, что себестоимость или подтверждение движения опираются на слабую или разваленную цепочку.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "qLnSZLaO7C6Wnv",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-010",
|
||||
"raw_question": "Есть ли материалы на 10 счёте, которые зависли так, что это уже выглядит не как забытая мелочь, а как системная дыра в логике участка.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "oKrKJofbhdQpGc",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-011",
|
||||
"raw_question": "Покажи позиции по материалам, где сумма сама по себе не пугает, но характер остатка и движения намекает, что потом это может аукнуться при разборе периода.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "g6Fzv_CQBSILRf",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-012",
|
||||
"raw_question": "Какие записи на 97 счёте больше всего похожи на историю, где документ завели, а дальше нормальной жизни у него не случилось: срок, списание и поведение записи друг другу противоречат.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "vO0aXk0LpG1Jnd",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-013",
|
||||
"raw_question": "Есть ли по РБП такие кейсы, где ошибка не в одной дате, а в том, что сама логика начала, окончания и длительности выглядит бухгалтерски сомнительно.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "1IgK6vkpj81btm",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-014",
|
||||
"raw_question": "Покажи объекты ОС, где карточка, срок и логика начисления амортизации выглядят так, будто объект поставили на учёт скорее “как-нибудь”, чем по нормальной модели.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "46ycjQ36kb1LDa",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-015",
|
||||
"raw_question": "Собери список самых подозрительных объектов ОС: не по сумме, а по риску того, что потом придётся долго объяснять, почему карточка и начисления не бьются между собой.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "partial",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "qNEsHD24JuSgQF",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-016",
|
||||
"raw_question": "Где у нас в целом по июню повторяется один и тот же тип проблемной связи, а не просто всплывают разовые косяки по разным участкам.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "q1S5H3DTSprheT",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-017",
|
||||
"raw_question": "Если смотреть на компанию как на единый контур, какие три зоны сейчас опаснее всего с точки зрения накопленного технического мусора в учёте, который может вылезти на закрытии.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "BBX_oFpDovY25m",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-018",
|
||||
"raw_question": "Разложи по приоритету не просто ошибки, а именно те места, где человеку потом придётся дольше всего вручную распутывать, что произошло и почему это не сошлось.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "bsjG2PBnXRhIwI",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-019",
|
||||
"raw_question": "Есть ли такие висящие истории, где по верхнему уровню всё выглядит терпимо, но если копнуть вглубь, видно, что несколько сущностей держатся на слабой или ложной связи.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 2,
|
||||
"in_scope_fragments": 2,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "GReq1033MnMBam",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-020",
|
||||
"raw_question": "Покажи мне не просто аномалии, а аномалии второго порядка: такие места, где отдельная ошибка не страшна, но в сочетании с другим участком уже начинает ломать картину периода.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": true,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "XfU4rc2JWv92TB",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-021",
|
||||
"raw_question": "Что сейчас в июне выглядит как история, которую бухгалтер отложил бы “на потом”, но именно из-за этого она может стать самой неприятной на закрытии.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "33mEeTPjTIryFa",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-022",
|
||||
"raw_question": "Если бы нужно было за один заход проверить только пять узлов учёта перед закрытием, какие пять ты бы выбрал и по каким объективным признакам.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "KnEFF9-NIFvoXF",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-023",
|
||||
"raw_question": "Есть ли кейсы, где один и тот же контрагент или объект всплывает сразу в нескольких подозрительных связях, и это уже похоже на концентратор проблем, а не на случайность.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "hodHj0_j7erCf3",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-024",
|
||||
"raw_question": "Покажи вопросы, которые можно решить только через причинную раскладку по документам и проводкам, а не просто через общий обзор подозрительных мест.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "DV_JPsG96AOEpz",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-025",
|
||||
"raw_question": "Есть ли в нашей базе куски, которые выглядят так, будто их кто-то руками “дособирал”, и теперь формально всё стоит, но смысловая логика учёта у этого шатается.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "9zgbcBw6rEvf_w",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-026",
|
||||
"raw_question": "Как в целом правильно отражать расходы будущих периодов по лицензиям и подпискам по российским правилам, если говорить вообще, а не по нашей базе.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 0,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "B0WyKNQBSUQcEp",
|
||||
"request_count_for_case": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-uv39LPHIUH",
|
||||
"timestamp": "2026-03-25T20:40:30.304Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Ax6feGtdH0l-gt",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "U16kNOzhP0YMSc",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,978 @@
|
||||
{
|
||||
"run_id": "eval-vHNdfS-Ccc",
|
||||
"timestamp": "2026-03-23T15:57:42.766Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": false,
|
||||
"prompt_version": "normalizer_v1_1_2",
|
||||
"dataset": {
|
||||
"source": "file",
|
||||
"file": "normalizer_eval_v1_1_30cases.json"
|
||||
},
|
||||
"cases_total": 30,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 96.67,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"baseline_metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 72.73,
|
||||
"route_hint_accuracy": 90.91,
|
||||
"causal_flag_accuracy": 81.82,
|
||||
"high_confidence_error_rate": 9.09
|
||||
},
|
||||
"baseline_delta": {
|
||||
"schema_validation_pass_rate": 0,
|
||||
"intent_class_accuracy": 23.94,
|
||||
"route_hint_accuracy": 9.09,
|
||||
"causal_flag_accuracy": 18.18,
|
||||
"high_confidence_error_rate": -9.09
|
||||
},
|
||||
"class_accuracy": {
|
||||
"cross_entity": {
|
||||
"total": 10,
|
||||
"passed": 10,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"heavy_analytical": {
|
||||
"total": 5,
|
||||
"passed": 5,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"drilldown_explain": {
|
||||
"total": 5,
|
||||
"passed": 5,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"rule_based_account_control": {
|
||||
"total": 5,
|
||||
"passed": 5,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"anomaly_probe": {
|
||||
"total": 2,
|
||||
"passed": 2,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"ambiguous_human_query": {
|
||||
"total": 1,
|
||||
"passed": 1,
|
||||
"accuracy_percent": 100
|
||||
},
|
||||
"period_close_risk": {
|
||||
"total": 2,
|
||||
"passed": 1,
|
||||
"accuracy_percent": 50
|
||||
}
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 30,
|
||||
"retries_used": 0,
|
||||
"guidance": {
|
||||
"forensic_calls_max": 10,
|
||||
"final_eval_calls_max": 30,
|
||||
"target_total_calls_max": 40,
|
||||
"hard_cap_calls_max": 45
|
||||
}
|
||||
},
|
||||
"mismatches": [
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||||
"trace_id": "YkYHshBB4NCoMa"
|
||||
}
|
||||
],
|
||||
"bad_confidence_cases": [],
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-001",
|
||||
"raw_question": "По каким поставщикам на конец июня не бьются взаиморасчеты, покажи документы, оплаты и хвосты.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "BU8964dx04zD_J",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-006",
|
||||
"raw_question": "По каким реализациям 90/62 хвосты не закрылись оплатой, разложи по цепочке документов.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "0BPtggSbkFlbBZ",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-003",
|
||||
"raw_question": "Где в июне не сходится 60/51: разложи по документу, оплате и закрывающему, чем подтверждается каждый шаг.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": true
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "hw-PNg2xcxJbi4",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-004",
|
||||
"raw_question": "Разложи по контрагентам цепочку: отгрузка -> оплата -> закрывающий, чтобы понять где рвется подтверждение.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "IUOuBvYZukJ1MN",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-005",
|
||||
"raw_question": "По поставщикам где повисло в цепочке поступление-оплата-закрытие по 60, покажи проблемные связки.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "9GSbmBW5rq2QWW",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-006",
|
||||
"raw_question": "Найди где по 62 не собралось: нужен разбор по документам, оплатам и проводкам с причинно-следственной цепочкой.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "h3RIys4qK8eq4H",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-007",
|
||||
"raw_question": "Покажи по июню все случаи когда реализация без оплаты и где в цепочке ошибка подтверждения.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "YoTIoXkQu_LXI9",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-008",
|
||||
"raw_question": "Сделай причинный разбор хвостов по 60: документ, оплата, проводка, закрывающий, где пошло криво.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "g6PKCrFwR5i8l3",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-009",
|
||||
"raw_question": "Почему у части покупателей не видно закрытия, разложи цепочку документов и оплат по июню.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "eZv5dJiLo1LftY",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-CE-010",
|
||||
"raw_question": "У кого из контрагентов в июне хвосты между 60 и банком, разложи по документам/оплатам/закрывающим.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "cross_entity",
|
||||
"actual_intent_class": "cross_entity",
|
||||
"expected_route_hint": "hybrid_store_plus_live",
|
||||
"actual_route_hint": "hybrid_store_plus_live",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": true,
|
||||
"needs_causal_chain": true,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "aIccf_QezPkgD4",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "fbKVmHy1EETElU",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "d4K6bAXInuPk9w",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-HA-003",
|
||||
"raw_question": "Собери топ-10 риск-зон учета по июню и приоритизируй, куда лезть сначала.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "UtQB4hEj5rX8-o",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-HA-004",
|
||||
"raw_question": "Дай обзорный риск-срез перед сдачей отчетности: где максимальная концентрация ошибок.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "l54TjB7oc5Qyjv",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-HA-005",
|
||||
"raw_question": "Сделай приоритизированный обзор ручных проверок по компании за июнь.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "heavy_analytical",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "Th8xfJlXssc6rT",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-003",
|
||||
"raw_question": "Покажи документ по номеру 000123 и строку проводки, нужен точный source-of-record.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "IzEts3dLpv8QBR",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "f5UwyNyjFPTyhG",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-003",
|
||||
"raw_question": "Покажи проводку по документу INV-2020-0615, нужна конкретная строка и источник.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "fbiSnFsRVlqC1O",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-004",
|
||||
"raw_question": "Дай точечный drilldown по документу №PAY-441 и его проводке по 51.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "f6YoMamYan-Dcd",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-DD-005",
|
||||
"raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "drilldown_explain",
|
||||
"actual_intent_class": "drilldown_explain",
|
||||
"expected_route_hint": "live_mcp_drilldown",
|
||||
"actual_route_hint": "live_mcp_drilldown",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": true,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": true,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "BEpk33zRSlK9rC",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-004",
|
||||
"raw_question": "По 97 счету проверь, где возможна ошибка дат начала и окончания списания.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "rule_based_account_control",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "H4Am6OSyIp7AdD",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-RB-002",
|
||||
"raw_question": "Проверь контрольные правила по ОС: где ошибки в сроках амортизации и учетной группе.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "rule_based_account_control",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "hUziWmv1gjYVzj",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-RB-003",
|
||||
"raw_question": "По 10 счету проверь где нарушены правила оценки остатков.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "rule_based_account_control",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "OdyARNHbgs0TVi",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-RB-004",
|
||||
"raw_question": "По НДС на 68.02 найди нарушения контрольных правил расчета за июнь.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "rule_based_account_control",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "nkfe3gFbX6KHxc",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-RB-005",
|
||||
"raw_question": "Проверь учетные настройки списания на 97 и покажи где высокий риск ручной ошибки.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "rule_based_account_control",
|
||||
"actual_intent_class": "rule_based_account_control",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "460rwjZiEgrXYR",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-005",
|
||||
"raw_question": "Есть ли аномальные материалы на счете 10, которые зависли и выглядят нелогично?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "anomaly_probe",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "hj-CYJdNY-H3W7",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-009",
|
||||
"raw_question": "Где у нас пахнет ручной ошибкой по июню?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "ambiguous_human_query",
|
||||
"actual_intent_class": "ambiguous_human_query",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": false,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "-0Dxa4G1t9dD6t",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-003",
|
||||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "period_close_risk",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": false,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "DY1FqJtqfNq6Ps",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-004",
|
||||
"raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "anomaly_probe",
|
||||
"actual_intent_class": "anomaly_probe",
|
||||
"expected_route_hint": "store_feature_risk",
|
||||
"actual_route_hint": "store_feature_risk",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": false,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "high",
|
||||
"trace_id": "5E2MCsQgze3XqD",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "V11-OT-005",
|
||||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"expected_intent_class": "period_close_risk",
|
||||
"actual_intent_class": "heavy_analytical",
|
||||
"expected_route_hint": "batch_refresh_then_store",
|
||||
"actual_route_hint": "batch_refresh_then_store",
|
||||
"expected_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false
|
||||
},
|
||||
"actual_requires": {
|
||||
"needs_cross_entity_join": false,
|
||||
"needs_causal_chain": false,
|
||||
"needs_exact_object_trace": false,
|
||||
"needs_ranking": true,
|
||||
"needs_anomaly_summary": true,
|
||||
"needs_runtime_truth": false,
|
||||
"needs_period_cut": true,
|
||||
"needs_evidence": false
|
||||
},
|
||||
"confidence_overall": "medium",
|
||||
"trace_id": "YkYHshBB4NCoMa",
|
||||
"request_count_for_case": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-w5euBjw0Va",
|
||||
"timestamp": "2026-03-24T08:28:53.002Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "o2_46OplZ51ysh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "HqmAe7AbP6VZFh",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "0TFOICc0l9afwf",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-wDfKUhOsoN",
|
||||
"timestamp": "2026-03-25T18:26:22.556Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Pyp9TQUbfI4qfo",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "6Bwd9cnb7KPFHL",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "vBSj8LlyW3YpNZ",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
{
|
||||
"run_id": "eval-wO885tbCgd",
|
||||
"timestamp": "2026-03-23T18:47:12.834Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": false,
|
||||
"prompt_version": "normalizer_v2_0_1",
|
||||
"schema_version": "v2_0_1",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 10
|
||||
},
|
||||
"cases_total": 10,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 4,
|
||||
"store_feature_risk": 6
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Какие реализации на конец периода выглядят так, будто они зависли и будут портить картину по выручке, если их не проверить заранее?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "39JMzDVF5alXJM",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи реализации, где хвост выглядит особенно неприятно: сумма не маленькая, возраст хвоста уже заметный, и при этом не видно нормального завершения цепочки.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "1quAW9woZFTC_a",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "По каким продажам на конец месяца видно, что бухгалтер потом будет долго распутывать, почему все это не сошлось нормально?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "5Ykxwl4SJl8Lm6",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-004",
|
||||
"raw_question": "Где банк и бухгалтерский контур, скорее всего, расходятся не по одной строке, а по паттерну, который уже начинает повторяться?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "dujfREyQcWG1p8",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-005",
|
||||
"raw_question": "Есть ли такие зависшие авансы, которые уже давно надо было либо закрыть, либо хотя бы перепроверить руками?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "WzXkwFwiGTtXKC",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-006",
|
||||
"raw_question": "Что сейчас лежит на 10 счете так, будто это уже давно надо было либо списать, либо хотя бы проверить, почему оно до сих пор висит?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "qwjznHv2OU7_YB",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-007",
|
||||
"raw_question": "Покажи позиции по материалам, где возможен эффект \"вроде сумма не огромная, но учетная логика выглядит криво\".",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "7qKuSI4SrCPJWr",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-008",
|
||||
"raw_question": "Есть ли такие расходы будущих периодов, которые заведены, но по ним не видно нормальной ежемесячной жизни, как будто запись повисла сама по себе?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "XxK59RCHy_vM5Z",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-009",
|
||||
"raw_question": "Покажи объекты ОС, где риск не в сумме, а в том, что карточка и логика начисления выглядят подозрительно и могут аукнуться позже.",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Eyq9tJ_zuOSOJb",
|
||||
"request_count_for_case": 1
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-010",
|
||||
"raw_question": "Есть ли движения по счету 51, которые выглядят корректно по сумме, но по смыслу оставляют после себя подозрительный хвост?",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "YcDoamdW0qAwy9",
|
||||
"request_count_for_case": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"run_id": "eval-xJm3dBJ6oZ",
|
||||
"timestamp": "2026-03-23T13:48:54.952Z",
|
||||
"cases_total": 10,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"intent_class_accuracy": 80,
|
||||
"route_hint_accuracy": 100,
|
||||
"causal_flag_accuracy": 100,
|
||||
"high_confidence_error_rate": 0
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "NQ-001",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "phW6TQF_zwo4ps"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-002",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "HFw3C2vcTq98Jl"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-003",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "SHAUEgGSDSpOnQ"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-004",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "psgRyAubKKbNIb"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-005",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "_8FofWSTg5zVKK"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-006",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "NVvbOsKjvEDCJ2"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-007",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "7gponohJuP_3ya"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-008",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "HdD4JkYZzN3jOX"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-009",
|
||||
"validation_passed": true,
|
||||
"intent_match": false,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "ApCNbWg0IG-BPE"
|
||||
},
|
||||
{
|
||||
"case_id": "NQ-010",
|
||||
"validation_passed": true,
|
||||
"intent_match": true,
|
||||
"route_match": true,
|
||||
"causal_flags_match": true,
|
||||
"trace_id": "Aram-Qxzh6zRSP"
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"run_id": "eval-yzcu57HH4b",
|
||||
"timestamp": "2026-03-24T10:31:40.300Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 33.33,
|
||||
"no_route_fragment_rate": 66.67,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 3
|
||||
},
|
||||
"route_distribution": {
|
||||
"store_feature_risk": 1,
|
||||
"no_route": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "TRMhN4t4D2WS7i",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "yEkGvuvoxsomMH",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "insufficient_specificity",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "ZGDpgDXvFCmoxB",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user