{ "run_id": "eval-xQU7mHgdgo", "timestamp": "2026-03-23T14:56:24.747Z", "mode": "single-pass-strict", "use_mock": false, "prompt_version": "normalizer_v1_1", "dataset": { "source": "file", "file": "normalizer_eval_v1_1_30cases.json" }, "cases_total": 30, "metrics": { "schema_validation_pass_rate": 100, "intent_class_accuracy": 93.33, "route_hint_accuracy": 96.67, "causal_flag_accuracy": 93.33, "high_confidence_error_rate": 0 }, "baseline_metrics": { "schema_validation_pass_rate": 100, "intent_class_accuracy": 72.73, "route_hint_accuracy": 90.91, "causal_flag_accuracy": 81.82, "high_confidence_error_rate": 9.09 }, "baseline_delta": { "schema_validation_pass_rate": 0, "intent_class_accuracy": 20.6, "route_hint_accuracy": 5.76, "causal_flag_accuracy": 11.51, "high_confidence_error_rate": -9.09 }, "class_accuracy": { "cross_entity": { "total": 10, "passed": 10, "accuracy_percent": 100 }, "heavy_analytical": { "total": 5, "passed": 5, "accuracy_percent": 100 }, "drilldown_explain": { "total": 5, "passed": 5, "accuracy_percent": 100 }, "rule_based_account_control": { "total": 5, "passed": 5, "accuracy_percent": 100 }, "anomaly_probe": { "total": 2, "passed": 2, "accuracy_percent": 100 }, "ambiguous_human_query": { "total": 1, "passed": 1, "accuracy_percent": 100 }, "period_close_risk": { "total": 2, "passed": 0, "accuracy_percent": 0 } }, "budget": { "requests_total": 30, "retries_used": 0, "guidance": { "forensic_calls_max": 10, "final_eval_calls_max": 30, "target_total_calls_max": 40, "hard_cap_calls_max": 45 } }, "mismatches": [ { "case_id": "NQ-008", "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "comment": "Causal flags are inconsistent with expected relationship depth.", "trace_id": "RFFanswt5Aeenl" }, { "case_id": "V11-DD-005", "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "comment": "Causal flags are inconsistent with expected relationship depth.", "trace_id": "pn6cHYsZRSWRqP" }, { "case_id": "V11-OT-003", "expected_intent_class": "period_close_risk", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.", "trace_id": "9HpW3Y0yLfRhxt" }, { "case_id": "V11-OT-004", "expected_intent_class": "anomaly_probe", "actual_intent_class": "anomaly_probe", "expected_route_hint": "store_feature_risk", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "comment": "Intent understood, but route_hint selected a weaker execution route.", "trace_id": "2ST0tr4E-gsBFR" }, { "case_id": "V11-OT-005", "expected_intent_class": "period_close_risk", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "comment": "Route chosen correctly, but intent_class drifted into a neighboring taxonomy bucket.", "trace_id": "hi28jOTc2uD0bn" } ], "bad_confidence_cases": [], "results": [ { "case_id": "NQ-001", "raw_question": "По каким поставщикам на конец июня не бьются взаиморасчеты, покажи документы, оплаты и хвосты.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "lU-dFgmFyY_1zE", "request_count_for_case": 1 }, { "case_id": "NQ-006", "raw_question": "По каким реализациям 90/62 хвосты не закрылись оплатой, разложи по цепочке документов.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "w7-mIJXTisCOhe", "request_count_for_case": 1 }, { "case_id": "V11-CE-003", "raw_question": "Где в июне не сходится 60/51: разложи по документу, оплате и закрывающему, чем подтверждается каждый шаг.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": true }, "confidence_overall": "medium", "trace_id": "jmiVUvY5wlLmet", "request_count_for_case": 1 }, { "case_id": "V11-CE-004", "raw_question": "Разложи по контрагентам цепочку: отгрузка -> оплата -> закрывающий, чтобы понять где рвется подтверждение.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": true }, "confidence_overall": "medium", "trace_id": "-flpLc9UUvJT27", "request_count_for_case": 1 }, { "case_id": "V11-CE-005", "raw_question": "По поставщикам где повисло в цепочке поступление-оплата-закрытие по 60, покажи проблемные связки.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "vWrXm3SYP2ItGv", "request_count_for_case": 1 }, { "case_id": "V11-CE-006", "raw_question": "Найди где по 62 не собралось: нужен разбор по документам, оплатам и проводкам с причинно-следственной цепочкой.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "r8CpSx9Rdge78-", "request_count_for_case": 1 }, { "case_id": "V11-CE-007", "raw_question": "Покажи по июню все случаи когда реализация без оплаты и где в цепочке ошибка подтверждения.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "NDSq9jbaVDaUWA", "request_count_for_case": 1 }, { "case_id": "V11-CE-008", "raw_question": "Сделай причинный разбор хвостов по 60: документ, оплата, проводка, закрывающий, где пошло криво.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "q1Zq0vuHRkBjWW", "request_count_for_case": 1 }, { "case_id": "V11-CE-009", "raw_question": "Почему у части покупателей не видно закрытия, разложи цепочку документов и оплат по июню.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "UdcI9mtkK4xfWi", "request_count_for_case": 1 }, { "case_id": "V11-CE-010", "raw_question": "У кого из контрагентов в июне хвосты между 60 и банком, разложи по документам/оплатам/закрывающим.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "cross_entity", "actual_intent_class": "cross_entity", "expected_route_hint": "hybrid_store_plus_live", "actual_route_hint": "hybrid_store_plus_live", "expected_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": true, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "evJtLH3_z8-ZBU", "request_count_for_case": 1 }, { "case_id": "NQ-002", "raw_question": "Сделай рейтинг самых рисковых хвостов перед закрытием периода за июнь.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "heavy_analytical", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "Ka9b2rPO_HhqT6", "request_count_for_case": 1 }, { "case_id": "NQ-007", "raw_question": "Что у нас выглядит самым проблемным перед закрытием июня, если смотреть на компанию в целом?", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "heavy_analytical", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "t6MU_mXpM_azgJ", "request_count_for_case": 1 }, { "case_id": "V11-HA-003", "raw_question": "Собери топ-10 риск-зон учета по июню и приоритизируй, куда лезть сначала.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "heavy_analytical", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "high", "trace_id": "EOC53b0jxaUlLD", "request_count_for_case": 1 }, { "case_id": "V11-HA-004", "raw_question": "Дай обзорный риск-срез перед сдачей отчетности: где максимальная концентрация ошибок.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "heavy_analytical", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "NN-gYZnET7xaPs", "request_count_for_case": 1 }, { "case_id": "V11-HA-005", "raw_question": "Сделай приоритизированный обзор ручных проверок по компании за июнь.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "heavy_analytical", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "high", "trace_id": "UeBm14bbSyD_OT", "request_count_for_case": 1 }, { "case_id": "NQ-003", "raw_question": "Покажи документ по номеру 000123 и строку проводки, нужен точный source-of-record.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "ADhuDhHEyeY_Yn", "request_count_for_case": 1 }, { "case_id": "NQ-008", "raw_question": "Покажи по банку документ №TRX-88 и связанную проводку по 51.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": false, "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "RFFanswt5Aeenl", "request_count_for_case": 1 }, { "case_id": "V11-DD-003", "raw_question": "Покажи проводку по документу INV-2020-0615, нужна конкретная строка и источник.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "PkTVhqFq9rM-hM", "request_count_for_case": 1 }, { "case_id": "V11-DD-004", "raw_question": "Дай точечный drilldown по документу №PAY-441 и его проводке по 51.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "tYJ_hZsKZU8rcO", "request_count_for_case": 1 }, { "case_id": "V11-DD-005", "raw_question": "Покажи карточку конкретной операции DOC-7781 и связанную проводку.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": false, "expected_intent_class": "drilldown_explain", "actual_intent_class": "drilldown_explain", "expected_route_hint": "live_mcp_drilldown", "actual_route_hint": "live_mcp_drilldown", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": true, "needs_causal_chain": false, "needs_exact_object_trace": true, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": true, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "pn6cHYsZRSWRqP", "request_count_for_case": 1 }, { "case_id": "NQ-004", "raw_question": "По 97 счету проверь, где возможна ошибка дат начала и окончания списания.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "rule_based_account_control", "actual_intent_class": "rule_based_account_control", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "1WWdojMzkPnh8b", "request_count_for_case": 1 }, { "case_id": "V11-RB-002", "raw_question": "Проверь контрольные правила по ОС: где ошибки в сроках амортизации и учетной группе.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "rule_based_account_control", "actual_intent_class": "rule_based_account_control", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "XuH8WPbONxqkKR", "request_count_for_case": 1 }, { "case_id": "V11-RB-003", "raw_question": "По 10 счету проверь где нарушены правила оценки остатков.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "rule_based_account_control", "actual_intent_class": "rule_based_account_control", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "ipXVpwKJxGTGcr", "request_count_for_case": 1 }, { "case_id": "V11-RB-004", "raw_question": "По НДС на 68.02 найди нарушения контрольных правил расчета за июнь.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "rule_based_account_control", "actual_intent_class": "rule_based_account_control", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "Aqo6w62BL7arj-", "request_count_for_case": 1 }, { "case_id": "V11-RB-005", "raw_question": "Проверь учетные настройки списания на 97 и покажи где высокий риск ручной ошибки.", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "rule_based_account_control", "actual_intent_class": "rule_based_account_control", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "nA6mic7NdLGmu8", "request_count_for_case": 1 }, { "case_id": "NQ-005", "raw_question": "Есть ли аномальные материалы на счете 10, которые зависли и выглядят нелогично?", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "anomaly_probe", "actual_intent_class": "anomaly_probe", "expected_route_hint": "store_feature_risk", "actual_route_hint": "store_feature_risk", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "yBdb2GdBpHLw_T", "request_count_for_case": 1 }, { "case_id": "NQ-009", "raw_question": "Где у нас пахнет ручной ошибкой по июню?", "validation_passed": true, "intent_match": true, "route_match": true, "causal_flags_match": true, "expected_intent_class": "ambiguous_human_query", "actual_intent_class": "ambiguous_human_query", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": false, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "JK3br_QDlyylUl", "request_count_for_case": 1 }, { "case_id": "V11-OT-003", "raw_question": "Перед закрытием периода что у нас может взорваться в последний день?", "validation_passed": true, "intent_match": false, "route_match": true, "causal_flags_match": true, "expected_intent_class": "period_close_risk", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": false, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "9HpW3Y0yLfRhxt", "request_count_for_case": 1 }, { "case_id": "V11-OT-004", "raw_question": "Где по июню выглядит подозрительно, но без точечного документа, просто дай зоны риска.", "validation_passed": true, "intent_match": true, "route_match": false, "causal_flags_match": true, "expected_intent_class": "anomaly_probe", "actual_intent_class": "anomaly_probe", "expected_route_hint": "store_feature_risk", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": false, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "2ST0tr4E-gsBFR", "request_count_for_case": 1 }, { "case_id": "V11-OT-005", "raw_question": "Че-то все криво на предзакрытии, где самые опасные места?", "validation_passed": true, "intent_match": false, "route_match": true, "causal_flags_match": true, "expected_intent_class": "period_close_risk", "actual_intent_class": "heavy_analytical", "expected_route_hint": "batch_refresh_then_store", "actual_route_hint": "batch_refresh_then_store", "expected_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false }, "actual_requires": { "needs_cross_entity_join": false, "needs_causal_chain": false, "needs_exact_object_trace": false, "needs_ranking": true, "needs_anomaly_summary": true, "needs_runtime_truth": false, "needs_period_cut": true, "needs_evidence": false }, "confidence_overall": "medium", "trace_id": "hi28jOTc2uD0bn", "request_count_for_case": 1 } ] }