1071 lines
38 KiB
JSON
1071 lines
38 KiB
JSON
{
|
||
"run_id": "eval-xQU7mHgdgo",
|
||
"timestamp": "2026-03-23T14:56:24.747Z",
|
||
"mode": "single-pass-strict",
|
||
"use_mock": false,
|
||
"prompt_version": "normalizer_v1_1",
|
||
"dataset": {
|
||
"source": "file",
|
||
"file": "normalizer_eval_v1_1_30cases.json"
|
||
},
|
||
"cases_total": 30,
|
||
"metrics": {
|
||
"schema_validation_pass_rate": 100,
|
||
"intent_class_accuracy": 93.33,
|
||
"route_hint_accuracy": 96.67,
|
||
"causal_flag_accuracy": 93.33,
|
||
"high_confidence_error_rate": 0
|
||
},
|
||
"baseline_metrics": {
|
||
"schema_validation_pass_rate": 100,
|
||
"intent_class_accuracy": 72.73,
|
||
"route_hint_accuracy": 90.91,
|
||
"causal_flag_accuracy": 81.82,
|
||
"high_confidence_error_rate": 9.09
|
||
},
|
||
"baseline_delta": {
|
||
"schema_validation_pass_rate": 0,
|
||
"intent_class_accuracy": 20.6,
|
||
"route_hint_accuracy": 5.76,
|
||
"causal_flag_accuracy": 11.51,
|
||
"high_confidence_error_rate": -9.09
|
||
},
|
||
"class_accuracy": {
|
||
"cross_entity": {
|
||
"total": 10,
|
||
"passed": 10,
|
||
"accuracy_percent": 100
|
||
},
|
||
"heavy_analytical": {
|
||
"total": 5,
|
||
"passed": 5,
|
||
"accuracy_percent": 100
|
||
},
|
||
"drilldown_explain": {
|
||
"total": 5,
|
||
"passed": 5,
|
||
"accuracy_percent": 100
|
||
},
|
||
"rule_based_account_control": {
|
||
"total": 5,
|
||
"passed": 5,
|
||
"accuracy_percent": 100
|
||
},
|
||
"anomaly_probe": {
|
||
"total": 2,
|
||
"passed": 2,
|
||
"accuracy_percent": 100
|
||
},
|
||
"ambiguous_human_query": {
|
||
"total": 1,
|
||
"passed": 1,
|
||
"accuracy_percent": 100
|
||
},
|
||
"period_close_risk": {
|
||
"total": 2,
|
||
"passed": 0,
|
||
"accuracy_percent": 0
|
||
}
|
||
},
|
||
"budget": {
|
||
"requests_total": 30,
|
||
"retries_used": 0,
|
||
"guidance": {
|
||
"forensic_calls_max": 10,
|
||
"final_eval_calls_max": 30,
|
||
"target_total_calls_max": 40,
|
||
"hard_cap_calls_max": 45
|
||
}
|
||
},
|
||
"mismatches": [
|
||
{
|
||
"case_id": "NQ-008",
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"comment": "Causal flags are inconsistent with expected relationship depth.",
|
||
"trace_id": "RFFanswt5Aeenl"
|
||
},
|
||
{
|
||
"case_id": "V11-DD-005",
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"comment": "Causal flags are inconsistent with expected relationship depth.",
|
||
"trace_id": "pn6cHYsZRSWRqP"
|
||
},
|
||
{
|
||
"case_id": "V11-OT-003",
|
||
"expected_intent_class": "period_close_risk",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||
"trace_id": "9HpW3Y0yLfRhxt"
|
||
},
|
||
{
|
||
"case_id": "V11-OT-004",
|
||
"expected_intent_class": "anomaly_probe",
|
||
"actual_intent_class": "anomaly_probe",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"comment": "Intent understood, but route_hint selected a weaker execution route.",
|
||
"trace_id": "2ST0tr4E-gsBFR"
|
||
},
|
||
{
|
||
"case_id": "V11-OT-005",
|
||
"expected_intent_class": "period_close_risk",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.",
|
||
"trace_id": "hi28jOTc2uD0bn"
|
||
}
|
||
],
|
||
"bad_confidence_cases": [],
|
||
"results": [
|
||
{
|
||
"case_id": "NQ-001",
|
||
"raw_question": "По каким поставщикам на конец июня не бьются взаиморасчеты, покажи документы, оплаты и хвосты.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "lU-dFgmFyY_1zE",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-006",
|
||
"raw_question": "По каким реализациям 90/62 хвосты не закрылись оплатой, разложи по цепочке документов.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "w7-mIJXTisCOhe",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-003",
|
||
"raw_question": "Где в июне не сходится 60/51: разложи по документу, оплате и закрывающему, чем подтверждается каждый шаг.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": true
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "jmiVUvY5wlLmet",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-004",
|
||
"raw_question": "Разложи по контрагентам цепочку: отгрузка -> оплата -> закрывающий, чтобы понять где рвется подтверждение.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": true
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "-flpLc9UUvJT27",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-005",
|
||
"raw_question": "По поставщикам где повисло в цепочке поступление-оплата-закрытие по 60, покажи проблемные связки.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "vWrXm3SYP2ItGv",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-006",
|
||
"raw_question": "Найди где по 62 не собралось: нужен разбор по документам, оплатам и проводкам с причинно-следственной цепочкой.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "r8CpSx9Rdge78-",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-007",
|
||
"raw_question": "Покажи по июню все случаи когда реализация без оплаты и где в цепочке ошибка подтверждения.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "NDSq9jbaVDaUWA",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-008",
|
||
"raw_question": "Сделай причинный разбор хвостов по 60: документ, оплата, проводка, закрывающий, где пошло криво.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "q1Zq0vuHRkBjWW",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-009",
|
||
"raw_question": "Почему у части покупателей не видно закрытия, разложи цепочку документов и оплат по июню.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "UdcI9mtkK4xfWi",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-CE-010",
|
||
"raw_question": "У кого из контрагентов в июне хвосты между 60 и банком, разложи по документам/оплатам/закрывающим.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "cross_entity",
|
||
"actual_intent_class": "cross_entity",
|
||
"expected_route_hint": "hybrid_store_plus_live",
|
||
"actual_route_hint": "hybrid_store_plus_live",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": true,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "evJtLH3_z8-ZBU",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-002",
|
||
"raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "heavy_analytical",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "Ka9b2rPO_HhqT6",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-007",
|
||
"raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "heavy_analytical",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "t6MU_mXpM_azgJ",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-HA-003",
|
||
"raw_question": "Собери топ-10 риск-зон учета по июню и приоритизируй, куда лезть сначала.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "heavy_analytical",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "high",
|
||
"trace_id": "EOC53b0jxaUlLD",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-HA-004",
|
||
"raw_question": "Дай обзорный риск-срез перед сдачей отчетности: где максимальная концентрация ошибок.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "heavy_analytical",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "NN-gYZnET7xaPs",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-HA-005",
|
||
"raw_question": "Сделай приоритизированный обзор ручных проверок по компании за июнь.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "heavy_analytical",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "high",
|
||
"trace_id": "UeBm14bbSyD_OT",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-003",
|
||
"raw_question": "Покажи документ по номеру 000123 и строку проводки, нужен точный source-of-record.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "ADhuDhHEyeY_Yn",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-008",
|
||
"raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": false,
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "RFFanswt5Aeenl",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-DD-003",
|
||
"raw_question": "Покажи проводку по документу INV-2020-0615, нужна конкретная строка и источник.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "PkTVhqFq9rM-hM",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-DD-004",
|
||
"raw_question": "Дай точечный drilldown по документу №PAY-441 и его проводке по 51.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "tYJ_hZsKZU8rcO",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-DD-005",
|
||
"raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": false,
|
||
"expected_intent_class": "drilldown_explain",
|
||
"actual_intent_class": "drilldown_explain",
|
||
"expected_route_hint": "live_mcp_drilldown",
|
||
"actual_route_hint": "live_mcp_drilldown",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": true,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": true,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": true,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "pn6cHYsZRSWRqP",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-004",
|
||
"raw_question": "По 97 счету проверь, где возможна ошибка дат начала и окончания списания.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "rule_based_account_control",
|
||
"actual_intent_class": "rule_based_account_control",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "1WWdojMzkPnh8b",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-RB-002",
|
||
"raw_question": "Проверь контрольные правила по ОС: где ошибки в сроках амортизации и учетной группе.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "rule_based_account_control",
|
||
"actual_intent_class": "rule_based_account_control",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "XuH8WPbONxqkKR",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-RB-003",
|
||
"raw_question": "По 10 счету проверь где нарушены правила оценки остатков.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "rule_based_account_control",
|
||
"actual_intent_class": "rule_based_account_control",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "ipXVpwKJxGTGcr",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-RB-004",
|
||
"raw_question": "По НДС на 68.02 найди нарушения контрольных правил расчета за июнь.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "rule_based_account_control",
|
||
"actual_intent_class": "rule_based_account_control",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "Aqo6w62BL7arj-",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-RB-005",
|
||
"raw_question": "Проверь учетные настройки списания на 97 и покажи где высокий риск ручной ошибки.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "rule_based_account_control",
|
||
"actual_intent_class": "rule_based_account_control",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "nA6mic7NdLGmu8",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-005",
|
||
"raw_question": "Есть ли аномальные материалы на счете 10, которые зависли и выглядят нелогично?",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "anomaly_probe",
|
||
"actual_intent_class": "anomaly_probe",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "store_feature_risk",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "yBdb2GdBpHLw_T",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "NQ-009",
|
||
"raw_question": "Где у нас пахнет ручной ошибкой по июню?",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "ambiguous_human_query",
|
||
"actual_intent_class": "ambiguous_human_query",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": false,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "JK3br_QDlyylUl",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-OT-003",
|
||
"raw_question": "Перед закрытием периода что у нас может взорваться в последний день?",
|
||
"validation_passed": true,
|
||
"intent_match": false,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "period_close_risk",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": false,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "9HpW3Y0yLfRhxt",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-OT-004",
|
||
"raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.",
|
||
"validation_passed": true,
|
||
"intent_match": true,
|
||
"route_match": false,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "anomaly_probe",
|
||
"actual_intent_class": "anomaly_probe",
|
||
"expected_route_hint": "store_feature_risk",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": false,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "2ST0tr4E-gsBFR",
|
||
"request_count_for_case": 1
|
||
},
|
||
{
|
||
"case_id": "V11-OT-005",
|
||
"raw_question": "Че-то все криво на предзакрытии, где самые опасные места?",
|
||
"validation_passed": true,
|
||
"intent_match": false,
|
||
"route_match": true,
|
||
"causal_flags_match": true,
|
||
"expected_intent_class": "period_close_risk",
|
||
"actual_intent_class": "heavy_analytical",
|
||
"expected_route_hint": "batch_refresh_then_store",
|
||
"actual_route_hint": "batch_refresh_then_store",
|
||
"expected_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false
|
||
},
|
||
"actual_requires": {
|
||
"needs_cross_entity_join": false,
|
||
"needs_causal_chain": false,
|
||
"needs_exact_object_trace": false,
|
||
"needs_ranking": true,
|
||
"needs_anomaly_summary": true,
|
||
"needs_runtime_truth": false,
|
||
"needs_period_cut": true,
|
||
"needs_evidence": false
|
||
},
|
||
"confidence_overall": "medium",
|
||
"trace_id": "hi28jOTc2uD0bn",
|
||
"request_count_for_case": 1
|
||
}
|
||
]
|
||
}
|