Этап 4 / Волна 16: смысловая изоляция РБП и ОС, фиксы лайв-ответов и добивка экспорта
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-0E46_gT0ds",
|
||||
"timestamp": "2026-03-28T14:52:43.266Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "j4C5JaGsdTX31w",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "EiQ_YNd0Z7R8dD",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-0yLbBfl8QU",
|
||||
"timestamp": "2026-03-28T14:16:11.152Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "zeDz4lWmOU-sXS",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "Igp7ZjWw22GURw",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-1yuAQTgSrA",
|
||||
"timestamp": "2026-03-28T14:52:43.542Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "dFofCt0krmlNxt",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "yuuGwyY-66-Acz",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-3lDLPY889T",
|
||||
"timestamp": "2026-03-28T14:17:31.550Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "QsfgBqSscHBfPL",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "lrWC9XpjJMN6AA",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-48cua6GzNX",
|
||||
"timestamp": "2026-03-28T17:57:44.019Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "60LijB0t8hI6Rv",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "tpPYPTRrOcZlBe",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-LbmAxVpEYt",
|
||||
"timestamp": "2026-03-28T14:17:29.773Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "THNkvQLVaISlq3",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "0kU7UfIyaxB3TA",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "ZCKIUbbI9qTMwd",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-LpNMUq6e64",
|
||||
"timestamp": "2026-03-28T14:52:41.774Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "jjxZOJBxrjPdsX",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "bYY62dikNsLECZ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "baezDULblEFYkL",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-Mk-Ep58vC0",
|
||||
"timestamp": "2026-03-28T17:57:43.813Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "p_blMcrrCin_H_",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "esnlDt05yjx_03",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-N-nY96elpX",
|
||||
"timestamp": "2026-03-28T14:23:36.202Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "7mDZkne0GBSe62",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "wAVEOezciqQrtm",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "t5shV55ZPr8wir",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-S-kDKKK4xO",
|
||||
"timestamp": "2026-03-28T14:17:31.390Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "kl3P7qQGw4BRHD",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "yn8Ku_R5880RIB",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-UNNoKia3JQ",
|
||||
"timestamp": "2026-03-28T17:57:42.190Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "MY7YLMfaYP3kVR",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "gpwnhjSvR3NClQ",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "WYxkGFarO1WeXH",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-X71VuszUM2",
|
||||
"timestamp": "2026-03-28T14:47:45.410Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "QacgRS_Ur2ayEH",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "qtDYC8oP9w1v9B",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-ZBwXbFcCs9",
|
||||
"timestamp": "2026-03-28T14:23:37.997Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "NMt3FqmnkEylaF",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "TNvDsELGZj_UDF",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-_XbcacYQdg",
|
||||
"timestamp": "2026-03-28T14:16:09.590Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "84nqlo6CKGkRZm",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "Z-6QeOaELWJ40H",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "fq5Nw3FI86Ldc1",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-bNqkZdQS9g",
|
||||
"timestamp": "2026-03-28T14:23:37.732Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "AnVPMdNwg-rf3T",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по счету 97",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "utvJRunZgTZrKu",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"run_id": "eval-c2aEXxAGeh",
|
||||
"timestamp": "2026-03-28T14:47:43.841Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 3
|
||||
},
|
||||
"cases_total": 3,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 33.33,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 33.33,
|
||||
"routed_fragment_rate": 66.67,
|
||||
"no_route_fragment_rate": 33.33,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 66.67,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 3,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 1,
|
||||
"no_route": 1,
|
||||
"batch_refresh_then_store": 1
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 1,
|
||||
"out_of_scope": 1,
|
||||
"clarification": 1
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь хвосты по поставщикам и разложи цепочку",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "r6cJKlGb3sAo1h",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Как вообще по ФСБУ",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 1,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "out_of_scope",
|
||||
"predicted_route_status": "no_route",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": "out_of_scope",
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "RwtT5cfOm8EQzm",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-003",
|
||||
"raw_question": "Покажи топ рисков за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": false,
|
||||
"scope_confidence": "low",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 0,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 1,
|
||||
"fallback_type": "clarification",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 0,
|
||||
"trace_id": "aTubcVPJxRPKKy",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-giqMXxi5Cr",
|
||||
"timestamp": "2026-03-28T14:16:11.355Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "lqgRK3iNNLJBPj",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "8F-6dbOWrrZ8DS",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"run_id": "eval-rbxdQ0Yd-g",
|
||||
"timestamp": "2026-03-28T14:47:45.752Z",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"schema_version": "v2_0_2",
|
||||
"dataset": {
|
||||
"source": "inline_raw_questions",
|
||||
"file": null,
|
||||
"raw_questions_count": 2
|
||||
},
|
||||
"cases_total": 2,
|
||||
"metrics": {
|
||||
"schema_validation_pass_rate": 100,
|
||||
"scope_detection_accuracy": null,
|
||||
"scope_in_scope_rate": 100,
|
||||
"multi_intent_detected_rate": 0,
|
||||
"clarification_required_rate": 0,
|
||||
"avg_fragments_per_message": 1,
|
||||
"out_of_scope_fragment_rate": 0,
|
||||
"routed_fragment_rate": 100,
|
||||
"no_route_fragment_rate": 0,
|
||||
"route_resolution_accuracy": null,
|
||||
"no_route_precision": null,
|
||||
"false_no_route_rate": null,
|
||||
"execution_state_consistency_rate": 100,
|
||||
"executable_with_soft_assumptions_rate": 100,
|
||||
"soft_assumption_used_fragment_rate": 100,
|
||||
"clarification_precision": null,
|
||||
"clarification_recall": null,
|
||||
"false_clarification_rate": null
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 0,
|
||||
"retries_used": 0
|
||||
},
|
||||
"clarification_eval": {
|
||||
"labeled_cases": 0,
|
||||
"true_positive": 0,
|
||||
"false_positive": 0,
|
||||
"false_negative": 0
|
||||
},
|
||||
"route_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0,
|
||||
"expected_routed_cases": 0,
|
||||
"no_route_true_positive": 0,
|
||||
"no_route_false_positive": 0
|
||||
},
|
||||
"scope_eval": {
|
||||
"labeled_cases": 0,
|
||||
"correct_cases": 0
|
||||
},
|
||||
"execution_state_eval": {
|
||||
"checks_total": 2,
|
||||
"checks_passed": 2
|
||||
},
|
||||
"route_distribution": {
|
||||
"hybrid_store_plus_live": 2
|
||||
},
|
||||
"fallback_distribution": {
|
||||
"none": 2
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case_id": "BQ-001",
|
||||
"raw_question": "Проверь счет 60 за июнь 2020",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "KRtUQA4OmuvZXo",
|
||||
"request_count_for_case": 0
|
||||
},
|
||||
{
|
||||
"case_id": "BQ-002",
|
||||
"raw_question": "Покажи риски по НДС и по закрытию",
|
||||
"validation_passed": true,
|
||||
"message_in_scope": true,
|
||||
"scope_confidence": "high",
|
||||
"contains_multiple_tasks": false,
|
||||
"fragments_total": 1,
|
||||
"in_scope_fragments": 1,
|
||||
"out_of_scope_fragments": 0,
|
||||
"unclear_fragments": 0,
|
||||
"fallback_type": "none",
|
||||
"predicted_route_status": "routed",
|
||||
"expected_route_status": null,
|
||||
"predicted_no_route_reason": null,
|
||||
"expected_no_route_reason": null,
|
||||
"predicted_clarification_required": false,
|
||||
"expected_clarification_required": null,
|
||||
"executable_with_soft_assumptions_fragments": 1,
|
||||
"trace_id": "6HH1OsFaNsR6hO",
|
||||
"request_count_for_case": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user