Initial import NDC_1C
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-8PwBx9qv",
|
||||
"run_timestamp": "2026-03-25T20:07:25.111Z",
|
||||
"baseline_run_id": "assistant-stage1-vZMOXLijGx",
|
||||
"current_run_id": "assistant-stage1-iK-YSOl5SR",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-vZMOXLijGx.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-iK-YSOl5SR.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.89,
|
||||
"current": 0.78,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.67,
|
||||
"current": 2.33,
|
||||
"delta": 1.66,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.33,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-TRANSLIT-QUERY: usefulness 0 -> 0 (delta 0)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-8PwBx9qv
|
||||
- baseline_run_id: assistant-stage1-vZMOXLijGx
|
||||
- current_run_id: assistant-stage1-iK-YSOl5SR
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.89 | 0.78 | -0.11 | improved |
|
||||
| accountant_actionability_score | 0.67 | 2.33 | 1.66 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.33 | 0 | unchanged |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-E8mQ87zU",
|
||||
"run_timestamp": "2026-03-25T20:41:09.111Z",
|
||||
"baseline_run_id": "assistant-stage1-TFFcspY4z8",
|
||||
"current_run_id": "assistant-stage1-RQbicMzAKf",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-TFFcspY4z8.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-RQbicMzAKf.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.78,
|
||||
"current": 0.78,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.78,
|
||||
"current": 2.67,
|
||||
"delta": 1.89,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.22,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-TRANSLIT-QUERY: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 1.25 -> 1.5 (delta 0.25)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-E8mQ87zU
|
||||
- baseline_run_id: assistant-stage1-TFFcspY4z8
|
||||
- current_run_id: assistant-stage1-RQbicMzAKf
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.78 | 0.78 | 0 | unchanged |
|
||||
| accountant_actionability_score | 0.78 | 2.67 | 1.89 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.22 | -0.11 | improved |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-OP6wunk-",
|
||||
"run_timestamp": "2026-03-25T20:09:33.482Z",
|
||||
"baseline_run_id": "assistant-stage1-Isj2Tscs4g",
|
||||
"current_run_id": "assistant-stage1-40oxOKsgjV",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-Isj2Tscs4g.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-40oxOKsgjV.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.89,
|
||||
"current": 0.78,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.67,
|
||||
"current": 2.33,
|
||||
"delta": 1.66,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.33,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-TRANSLIT-QUERY: usefulness 0 -> 0 (delta 0)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-OP6wunk-
|
||||
- baseline_run_id: assistant-stage1-Isj2Tscs4g
|
||||
- current_run_id: assistant-stage1-40oxOKsgjV
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.89 | 0.78 | -0.11 | improved |
|
||||
| accountant_actionability_score | 0.67 | 2.33 | 1.66 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.33 | 0 | unchanged |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-b78uzAhM",
|
||||
"run_timestamp": "2026-03-25T20:41:15.635Z",
|
||||
"baseline_run_id": "assistant-stage1-vAXHvm-KsF",
|
||||
"current_run_id": "assistant-stage1-wiKJXmxb22",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-vAXHvm-KsF.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-wiKJXmxb22.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.78,
|
||||
"current": 0.78,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.78,
|
||||
"current": 2.67,
|
||||
"delta": 1.89,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.22,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-TRANSLIT-QUERY: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 1.25 -> 1.5 (delta 0.25)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-b78uzAhM
|
||||
- baseline_run_id: assistant-stage1-vAXHvm-KsF
|
||||
- current_run_id: assistant-stage1-wiKJXmxb22
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.78 | 0.78 | 0 | unchanged |
|
||||
| accountant_actionability_score | 0.78 | 2.67 | 1.89 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.22 | -0.11 | improved |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-qMhQrePC",
|
||||
"run_timestamp": "2026-03-25T20:44:23.281Z",
|
||||
"baseline_run_id": "assistant-stage1-yZwiFRQJyf",
|
||||
"current_run_id": "assistant-stage1-CwSTF_hjKz",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:/1C/NDC_1C/llm_normalizer/reports/assistant-stage1-yZwiFRQJyf.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-CwSTF_hjKz.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.78,
|
||||
"current": 0.78,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 2.33,
|
||||
"current": 2.67,
|
||||
"delta": 0.34,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.22,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 1,
|
||||
"unchanged": 8,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-TRANSLIT-QUERY: usefulness 0 -> 1.5 (delta 1.5)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 2.25 -> 2.25 (delta 0)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 1.5 -> 1.5 (delta 0)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 1.5 -> 1.5 (delta 0)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 1.5 -> 1.5 (delta 0)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 1.5 -> 1.5 (delta 0)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0.5 -> 0.5 (delta 0)",
|
||||
"S1-MULTI-INTENT: usefulness 1.5 -> 1.5 (delta 0)",
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 1.5 -> 1.5 (delta 0)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-qMhQrePC
|
||||
- baseline_run_id: assistant-stage1-yZwiFRQJyf
|
||||
- current_run_id: assistant-stage1-CwSTF_hjKz
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.78 | 0.78 | 0 | unchanged |
|
||||
| accountant_actionability_score | 2.33 | 2.67 | 0.34 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.22 | -0.11 | improved |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 1
|
||||
- unchanged: 8
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-u32A9CuI",
|
||||
"run_timestamp": "2026-03-25T20:07:36.336Z",
|
||||
"baseline_run_id": "assistant-stage1-TnhIS4Qc6e",
|
||||
"current_run_id": "assistant-stage1-ZQ5JDEUUkW",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-TnhIS4Qc6e.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-ZQ5JDEUUkW.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.89,
|
||||
"current": 0.78,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.67,
|
||||
"current": 2.33,
|
||||
"delta": 1.66,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.33,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-TRANSLIT-QUERY: usefulness 0 -> 0 (delta 0)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-u32A9CuI
|
||||
- baseline_run_id: assistant-stage1-TnhIS4Qc6e
|
||||
- current_run_id: assistant-stage1-ZQ5JDEUUkW
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.89 | 0.78 | -0.11 | improved |
|
||||
| accountant_actionability_score | 0.67 | 2.33 | 1.66 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.33 | 0 | unchanged |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-up7MQKOB",
|
||||
"run_timestamp": "2026-03-25T20:49:23.293Z",
|
||||
"baseline_run_id": "assistant-stage1-3cX10PNH8P",
|
||||
"current_run_id": "assistant-stage1-0XLmwlePaE",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-3cX10PNH8P.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-0XLmwlePaE.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.78,
|
||||
"current": 0.78,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.78,
|
||||
"current": 2.67,
|
||||
"delta": 1.89,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.22,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-TRANSLIT-QUERY: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 1.25 -> 1.5 (delta 0.25)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-up7MQKOB
|
||||
- baseline_run_id: assistant-stage1-3cX10PNH8P
|
||||
- current_run_id: assistant-stage1-0XLmwlePaE
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.78 | 0.78 | 0 | unchanged |
|
||||
| accountant_actionability_score | 0.78 | 2.67 | 1.89 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.22 | -0.11 | improved |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-y5A2jQKD",
|
||||
"run_timestamp": "2026-03-25T20:40:30.264Z",
|
||||
"baseline_run_id": "assistant-stage1-iRPNHs3DIw",
|
||||
"current_run_id": "assistant-stage1-oIPy2jOY3I",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-iRPNHs3DIw.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-oIPy2jOY3I.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.56,
|
||||
"delta": -0.11,
|
||||
"trend": "weakened"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.78,
|
||||
"current": 0.78,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.78,
|
||||
"current": 2.67,
|
||||
"delta": 1.89,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0,
|
||||
"delta": -0.33,
|
||||
"trend": "improved"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0,
|
||||
"delta": -0.25,
|
||||
"trend": "improved"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-TRANSLIT-QUERY: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 1.25 -> 1.5 (delta 0.25)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-y5A2jQKD
|
||||
- baseline_run_id: assistant-stage1-iRPNHs3DIw
|
||||
- current_run_id: assistant-stage1-oIPy2jOY3I
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.56 | -0.11 | weakened |
|
||||
| generic_explanation_rate | 0.78 | 0.78 | 0 | unchanged |
|
||||
| accountant_actionability_score | 0.78 | 2.67 | 1.89 | improved |
|
||||
| false_confidence_rate | 0.33 | 0 | -0.33 | improved |
|
||||
| broad_answer_rate | 0.25 | 0 | -0.25 | improved |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_comparison_v0_1",
|
||||
"comparison_id": "assistant-compare-z9MUKOjr",
|
||||
"run_timestamp": "2026-03-25T20:09:40.077Z",
|
||||
"baseline_run_id": "assistant-stage1-Kzrrn6Ckh2",
|
||||
"current_run_id": "assistant-stage1-yZwiFRQJyf",
|
||||
"eval_target": "assistant_stage1",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"baseline_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-Kzrrn6Ckh2.json",
|
||||
"current_report_file": "X:\\1C\\NDC_1C\\llm_normalizer\\reports\\assistant-stage1-yZwiFRQJyf.json",
|
||||
"metric_deltas": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"baseline": 0.67,
|
||||
"current": 0.67,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"baseline": 0.89,
|
||||
"current": 0.78,
|
||||
"delta": -0.11,
|
||||
"trend": "improved"
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"baseline": 0.67,
|
||||
"current": 2.33,
|
||||
"delta": 1.66,
|
||||
"trend": "improved"
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"baseline": 0.33,
|
||||
"current": 0.33,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"baseline": 0.25,
|
||||
"current": 0.25,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"baseline": 0,
|
||||
"current": 0,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"baseline": 3,
|
||||
"current": 3,
|
||||
"delta": 0,
|
||||
"trend": "unchanged"
|
||||
}
|
||||
},
|
||||
"scenario_notes_summary": {
|
||||
"improved": 8,
|
||||
"unchanged": 1,
|
||||
"weakened": 0
|
||||
},
|
||||
"scenario_notes": {
|
||||
"improved": [
|
||||
"S1-51-WRONG-CLOSE-TYPE: usefulness 1.5 -> 2.25 (delta 0.75)",
|
||||
"S1-60-SUPPLIER-TAILS: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-97-LIFECYCLE-ANOMALY: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-OS-CARD-VS-CHARGES: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION: usefulness 0 -> 1.5 (delta 1.5)",
|
||||
"S1-PERIOD-CLOSE-IMPACT: usefulness 0 -> 0.5 (delta 0.5)",
|
||||
"S1-MULTI-INTENT: usefulness 0.75 -> 1.5 (delta 0.75)",
|
||||
"S1-FOLLOWUP-INVESTIGATION: usefulness 0.75 -> 1.5 (delta 0.75)"
|
||||
],
|
||||
"unchanged": [
|
||||
"S1-TRANSLIT-QUERY: usefulness 0 -> 0 (delta 0)"
|
||||
],
|
||||
"weakened": []
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Baseline vs Current"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# Assistant Stage 1 Baseline vs Current
|
||||
|
||||
- comparison_id: assistant-compare-z9MUKOjr
|
||||
- baseline_run_id: assistant-stage1-Kzrrn6Ckh2
|
||||
- current_run_id: assistant-stage1-yZwiFRQJyf
|
||||
- suite_version: 0.1.0
|
||||
|
||||
## Metric Deltas
|
||||
|
||||
| Metric | Baseline | Current | Delta | Trend |
|
||||
|---|---:|---:|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 0.67 | 0 | unchanged |
|
||||
| generic_explanation_rate | 0.89 | 0.78 | -0.11 | improved |
|
||||
| accountant_actionability_score | 0.67 | 2.33 | 1.66 | improved |
|
||||
| false_confidence_rate | 0.33 | 0.33 | 0 | unchanged |
|
||||
| broad_answer_rate | 0.25 | 0.25 | 0 | unchanged |
|
||||
| mechanism_specificity_score | 0 | 0 | 0 | unchanged |
|
||||
| followup_context_retention_score | 3 | 3 | 0 | unchanged |
|
||||
|
||||
## Scenario Notes Summary
|
||||
|
||||
- improved: 8
|
||||
- unchanged: 1
|
||||
- weakened: 0
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1--ih_Qe9New",
|
||||
"run_timestamp": "2026-03-25T20:09:40.271Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "A1W4H0y7h5v77h",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "s5PCIS6w4G-eWI",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "4F-y--QLBC7Jy8",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "eCRipzC-_8PYWT",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "1HdH7tdgBmmaWY",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "gC9rwzybPGJVT2",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-MULTI-INTENT",
|
||||
"trace_id": "Vfs6O-_30dwuGt",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "ZxBa3wtLOjw1gx",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:40.271Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1--ih_Qe9New-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "F0x7BIHG40JkLX",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1--ih_Qe9New
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:40.271Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-0XLmwlePaE",
|
||||
"run_timestamp": "2026-03-25T20:49:23.292Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "lw79az3y6Wt33z",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "Vqgsw0brhXciM5",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "H08oFxd4AbvwzA",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "np1B0KlHUo1xsZ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "W357_Wwu2j4ebR",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "F4R-0LQjmVha_Z",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-MULTI-INTENT",
|
||||
"trace_id": "aalYVGk6UsL8mb",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "RgeT-XKzZW5_Bd",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.292Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-0XLmwlePaE-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "HkVw70kzh2-Eus",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-0XLmwlePaE
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:23.292Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-3cX10PNH8P",
|
||||
"run_timestamp": "2026-03-25T20:49:23.170Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 0.78,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "b8RhejxJnqCIk8",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "M4Ayk6UUo5rRp5",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "alpcxn7XE5gHYo",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "LzgwRf6kZ227h0",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "ITcG7DAhn0m289",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "u-5Sn0lQzzzS6L",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-MULTI-INTENT",
|
||||
"trace_id": "EjcMDlHTU-QqG3",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "e1ZPRQdnnuLzz5",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.170Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-3cX10PNH8P-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "gb-zPkVEyQqfYh",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": []
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-3cX10PNH8P
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:23.170Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.78 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-3sfiBZhUlN",
|
||||
"run_timestamp": "2026-03-25T20:41:15.356Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "qzySKd-bvPd8-n",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "EyhUgtx8tKe3IM",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "DLTK8WGkLpufYg",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "jsLSwL6VuV8aau",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "ur862FZL2uMOWz",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "qCZhX8P7VAxZzm",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-MULTI-INTENT",
|
||||
"trace_id": "PSI9UV27WM-41V",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "Ut3MHTc9LLQsMv",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.356Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-3sfiBZhUlN-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "_khngTqytzvo-v",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-3sfiBZhUlN
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:15.356Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-40oxOKsgjV",
|
||||
"run_timestamp": "2026-03-25T20:09:33.481Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "yMV2mUDyF69K42",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "_2CLmkNVYZKVWZ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "OkAIMQJPcJgLNP",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "cPtUdlXg7BRQug",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "rZZpTk1HrhNGxn",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "CHXoBd1VUvkCPo",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-MULTI-INTENT",
|
||||
"trace_id": "mKfMKbjjgYMgOy",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "c_6cifgtVV94y_",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.481Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-40oxOKsgjV-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "RWwtK0ss_tgK1X",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-40oxOKsgjV
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.481Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,723 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-4vbBmK39Yz",
|
||||
"run_timestamp": "2026-03-25T20:40:29.863Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.56,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "rZgojUQ3UByauS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "UYaJ6tyG09-L1Y",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "hQ51-aV-jtQbiz",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "B4siXjjruFoCwc",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "tTBlTfjpVmEcgi",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "-NEZ0uOQOW9WyO",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-MULTI-INTENT",
|
||||
"trace_id": "KktqI7ccvdfLYr",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "V_A14PmgnaXvfh",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:29.863Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-4vbBmK39Yz-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "F7qnhwWzraEd6K",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-4vbBmK39Yz
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:40:29.863Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.56 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-9DO0gB2Bng",
|
||||
"run_timestamp": "2026-03-25T20:49:22.891Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "Klh07nGhLBGVLO",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "Xim2CgCve6WqX4",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "G_Kz2kl5bN59LX",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "Fq9rMeIX3mmsbf",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "ZqFrVETKGb5Aba",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "x5BowA8qalaAuC",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-MULTI-INTENT",
|
||||
"trace_id": "aZoN1G8tu3FvrC",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "PwVtE68KiXIqyp",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:22.891Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-9DO0gB2Bng-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "_Xii9RIbEXyemn",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-9DO0gB2Bng
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:22.891Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-CwSTF_hjKz",
|
||||
"run_timestamp": "2026-03-25T20:44:23.279Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": null
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "61EczbrtEBhrWO",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "TYuVNiLcSTzCse",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "C947zGx925fyxE",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "JjDd4Lv6PSDked",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "VfzLPBKfbh0Sts",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "13SLIRkjDoGXwd",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-MULTI-INTENT",
|
||||
"trace_id": "EsTccuTKGhBYEK",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "UCE8ygBqN7mhVS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:44:23.279Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-CwSTF_hjKz-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "azvcIduuD7gFjB",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-CwSTF_hjKz
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:44:23.279Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-DRAO_THFxW",
|
||||
"run_timestamp": "2026-03-25T20:09:33.671Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "xxhnMo17gZrFtj",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "ps9I1V65IYH8Ey",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "BaVAt1SxNC918D",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "YtlnB8sK8xlLbb",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "2hNrKp0r64DT2p",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "XuQOHteJ-e_GGg",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-MULTI-INTENT",
|
||||
"trace_id": "kLUIxGTZd8edWU",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "tCYUCgsJ6_dmEK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.671Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-DRAO_THFxW-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "SlYP1ntK5FXWE2",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-DRAO_THFxW
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.671Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-FNlfUfeN4Z",
|
||||
"run_timestamp": "2026-03-25T20:09:39.866Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.866Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-FNlfUfeN4Z-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "nQKSLSNGEaT8iK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.866Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-FNlfUfeN4Z-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "YPKPWRxFFzGpok",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-FNlfUfeN4Z
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:39.866Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-HfaWvFRZnj",
|
||||
"run_timestamp": "2026-03-25T20:09:33.108Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "5jkBowy7Tp-egM",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "FBDMHngKd01gUr",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "MkOIzEbRgQAuyk",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "3hhFWYCVjvUutC",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "fcrKVoYMtprFFm",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "KaJI0Zt28CkwRv",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-MULTI-INTENT",
|
||||
"trace_id": "azuq8DGmvYqZO_",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "xGlgo4mZeXZnJq",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.108Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-HfaWvFRZnj-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "jmwaSB5JyTykDw",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-HfaWvFRZnj
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.108Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,711 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-Isj2Tscs4g",
|
||||
"run_timestamp": "2026-03-25T20:09:33.375Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.89,
|
||||
"accountant_actionability_score": 0.67,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "QeOnAiybT1ZobO",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "iVPHVLDLPbKDkS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "_aNH_Y7ja7jBF5",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "GSPoIqD8iWGPQp",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "C4t0JExGI9EEvk",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "cak-1d34TwpLuW",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-MULTI-INTENT",
|
||||
"trace_id": "QjlFlDfowf3Ppk",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "rRSBqbOwmJBAD2",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.375Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-Isj2Tscs4g-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "YDR8S1lQ5ZqzTW",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-Isj2Tscs4g
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.375Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.89 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.67 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,711 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-Kzrrn6Ckh2",
|
||||
"run_timestamp": "2026-03-25T20:09:39.962Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.89,
|
||||
"accountant_actionability_score": 0.67,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "PpWF5gKEdg5gtL",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "mW9UZ_QhMfHJ-q",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "2kmVGM4EI-dhtZ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "NQEEwucbet6MU-",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "b67oSDcG6SbOIk",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "WPfOYpbWW_Ol3I",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-MULTI-INTENT",
|
||||
"trace_id": "ezKKeqfg-NTLni",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "q5J5rShr3Ciu5V",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.962Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-Kzrrn6Ckh2-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "WYkO4ja1wgG88K",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-Kzrrn6Ckh2
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:39.962Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.89 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.67 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-LEzIhAe8V7",
|
||||
"run_timestamp": "2026-03-25T20:41:08.847Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "veB-kJSBVB4uTg",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "K05D5f5Xto4Ldc",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "JtMvQ-7c3mhOqB",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "u2lm58SHxYytCK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "fb6fDsr_YihS4k",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "eCnPEl7VFyGa0y",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-MULTI-INTENT",
|
||||
"trace_id": "cyX4oV9rH-uVeV",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "BiBtk0MkSsRKbh",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.847Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-LEzIhAe8V7-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "mWrTZFKHQhdvp_",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-LEzIhAe8V7
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:08.847Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-MIk0wJSOdI",
|
||||
"run_timestamp": "2026-03-25T20:07:36.498Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "LxONhczedOJGyj",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "PBQSqmiFfgYQiX",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "rQQJgPoJm1scxt",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "EsPQlHVWNLHUjD",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "KE529FpSuPXZr2",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "eczUksY3rqbPfb",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-MULTI-INTENT",
|
||||
"trace_id": "WAP1cFUgevHlcH",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "5XXb_D-8JAb4gw",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.498Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-MIk0wJSOdI-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "acAK8mgGXDg2cn",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-MIk0wJSOdI
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:36.498Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-MRFP57Dxr5",
|
||||
"run_timestamp": "2026-03-25T20:09:39.680Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "juxAWhfazLAY2D",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "bRBRmsx0L4FDCc",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "Z7FOgwFIbTG2P5",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "4GW9oV1-fEdL0Q",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "vDxkkQfpqDBFPU",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "CAgrr_IWlHBMQj",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-MULTI-INTENT",
|
||||
"trace_id": "g5yWZTaqaMrVIR",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "v155AsDpWeHhc9",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.680Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-MRFP57Dxr5-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "0dOeJk8AU2ohHb",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-MRFP57Dxr5
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:39.680Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-Mo6uxBZmrk",
|
||||
"run_timestamp": "2026-03-25T20:07:24.881Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.881Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-Mo6uxBZmrk-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "wnKLXsvVD3JzwD",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.881Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-Mo6uxBZmrk-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "UB8FKCI1fKgLkf",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-Mo6uxBZmrk
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:24.881Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-N1Tn9_PAM0",
|
||||
"run_timestamp": "2026-03-25T20:07:36.072Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "CUcnb815Dt1tjX",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "qNhKTsVnfXuf_W",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "bQUfCHX5WTQ8dB",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "B5DLLe80Y4iBck",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "xkLFYFAiOwJ03L",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "XMDf2u2F7L7-hu",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-MULTI-INTENT",
|
||||
"trace_id": "wOJATBZrVI3kat",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "6I40ITQxXUGtHq",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.072Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-N1Tn9_PAM0-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "hOwmEf0G0j7VZL",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-N1Tn9_PAM0
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:36.072Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-P92vpGQDdU",
|
||||
"run_timestamp": "2026-03-25T20:49:23.476Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "LgzSagi-J6zSHg",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "nzeDbTU6Wsa8D0",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "EmeQqGfAQ9b_fI",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "r6uxdMyKl-wbYK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "CurBfjR6aG9cIl",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "NNCKtuZiRKFVhq",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-MULTI-INTENT",
|
||||
"trace_id": "ZqVUko9X3YKtx4",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "gKmBiU1ig0RqL7",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.476Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-P92vpGQDdU-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "eGjSxWBusOr6u1",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-P92vpGQDdU
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:23.476Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-PSlm1B9wJL",
|
||||
"run_timestamp": "2026-03-25T20:07:25.288Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "jAGdpYVQSorWHQ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "o3rvu_K0yXRuAo",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "dQKZKG4-oOwQSi",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "qhc9pf9mDFrss-",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "YstgH7SofYgAL_",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "Ssd9D2tDSFU9xL",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-MULTI-INTENT",
|
||||
"trace_id": "L-K_DRp96zPVIN",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "Tc1yF-Q4wvGiT_",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.288Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-PSlm1B9wJL-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "ljOsTTv0IDMvwh",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-PSlm1B9wJL
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:25.288Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-PYoaIYdarC",
|
||||
"run_timestamp": "2026-03-25T20:07:35.963Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "5yg5GipDpEFRWN",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "0aN4_CfAlmEJZS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "xl4TMNbFA7cMZV",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "1nKtKSnIoq8mZD",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "8ONAsRUyP_d3jl",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "OImTFxMNI53jjF",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-MULTI-INTENT",
|
||||
"trace_id": "MKacWI13M91vAt",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "zXbTZPToePkjDr",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:35.963Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-PYoaIYdarC-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "KKGPUniioDCiEH",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-PYoaIYdarC
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:35.963Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-RQbicMzAKf",
|
||||
"run_timestamp": "2026-03-25T20:41:09.110Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.109Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "ct22qVSF_uYvC4",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "BHrrPHwFgwb3mf",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "qdEymWE3_jGksu",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "h2f_PVDq6bhLbK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "LJHTTb5qbM8N4c",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "TeZKbfAcItYOXW",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-MULTI-INTENT",
|
||||
"trace_id": "S4pXVKnJ5qi1Bq",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "PeW3jSCu-kXoRL",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.110Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-RQbicMzAKf-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "cCV2pwzGqGC3Ez",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-RQbicMzAKf
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:09.110Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-TFFcspY4z8",
|
||||
"run_timestamp": "2026-03-25T20:41:09.001Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 0.78,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "xioMH6-VXoPEz2",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "aOlhTFa3TplW74",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "2r4NruKL2gHvFU",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "gAg7MSIMJNOjyS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "3D8Fl2jSx9Ykch",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "QUvEdHSkAcsQuB",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-MULTI-INTENT",
|
||||
"trace_id": "PZTO5N_vSiQzd5",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "Ab-voWbVUoYkvS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.001Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-TFFcspY4z8-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "d4WErhAxNrffkZ",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": []
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-TFFcspY4z8
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:09.001Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.78 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,711 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-TnhIS4Qc6e",
|
||||
"run_timestamp": "2026-03-25T20:07:36.235Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.89,
|
||||
"accountant_actionability_score": 0.67,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "k1cYS90q5LqJ6i",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "uOluninDv5--WJ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "hHMN4pkk1wwMOf",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "JMfeLONKf4q56Z",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "nfFeCIbuB_qTPm",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "SgKjX8XlU_odU3",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-MULTI-INTENT",
|
||||
"trace_id": "lVOQhME0A1J0Ho",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "SOlDuJ0Jzg1d_Q",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.235Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-TnhIS4Qc6e-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "29wu_4ZadhYoCZ",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-TnhIS4Qc6e
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:36.235Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.89 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.67 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-TsMcfe2olY",
|
||||
"run_timestamp": "2026-03-25T20:40:30.036Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.036Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-TsMcfe2olY-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "9yOgfSgUHV_CVX",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.036Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-TsMcfe2olY-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "J9huThkGGtlSLN",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-TsMcfe2olY
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:40:30.036Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-UgHD_HBHT6",
|
||||
"run_timestamp": "2026-03-25T20:41:15.245Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "76lS2Bgq9F4JYs",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "kZtQZU2dHmeN-_",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "LDSxyaWfXj3_na",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "beoLrObEkr6zVh",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "ELo1o0ft8eGDLs",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "2W9B1sn-_93OgR",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-MULTI-INTENT",
|
||||
"trace_id": "uDmSWaQRyK46Pz",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "o4ByIjtMqLP_tw",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:15.244Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-UgHD_HBHT6-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "6FtAiU0vXfbKtZ",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-UgHD_HBHT6
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:15.245Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-W7qf0mZpA4",
|
||||
"run_timestamp": "2026-03-25T20:07:36.143Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.143Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-W7qf0mZpA4-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "jZ0OuqMEF0IQto",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.143Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-W7qf0mZpA4-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "A8h4B191vpwnkI",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-W7qf0mZpA4
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:36.143Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-ZQ5JDEUUkW",
|
||||
"run_timestamp": "2026-03-25T20:07:36.335Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "bL14PmRFYeGSfe",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "7rE4nkSmPUhULh",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "c5--A1CtkmZbB-",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "Y29maaZkRgiDNL",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "1OYxBdR6-5_Oj3",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "wHzpMY-kCgHNDM",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-MULTI-INTENT",
|
||||
"trace_id": "o6mIfFtFZpFGWu",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "2-FKPp0DNcmoVO",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:36.335Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-ZQ5JDEUUkW-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "ftWhl0nFuzeVCX",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-ZQ5JDEUUkW
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:36.335Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-_BNDWl91YW",
|
||||
"run_timestamp": "2026-03-25T20:41:09.344Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "Aa88osNKv3sek_",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "sMZ567regbbxuV",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "ZRFyn69TIAoVJ6",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "dIhnzqnqrk_Enm",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "hdM_306rP15fbm",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "wPuU7MfXYTC8Li",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-MULTI-INTENT",
|
||||
"trace_id": "de9jCLWS6gU129",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "b0as4LHD18kMEg",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:09.344Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-_BNDWl91YW-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "m7V62nIpP-QblY",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-_BNDWl91YW
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:09.344Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-aR_DtyIOU3",
|
||||
"run_timestamp": "2026-03-25T20:07:24.691Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "GQU1n5eIHBHEzr",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "56FXUe3YdZ5dMU",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "XTCrn_rTovUe7s",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "pMbu_nyv7bM8pm",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "qup7v7d8TciFPK",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "JQMNB4SBymHlaE",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-MULTI-INTENT",
|
||||
"trace_id": "5s610QkX-sixF9",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "nFJGSwi8nb_KfS",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:24.690Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-aR_DtyIOU3-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "owXorUJGLKwKrw",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-aR_DtyIOU3
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:24.691Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-bXmaw8PKnI",
|
||||
"run_timestamp": "2026-03-25T20:49:23.009Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "A9m07Dg-TjEWpx",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "4zjzUJTyT8B1es",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "qKDEUSYyMhS4l_",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "BJOjv2jnSL_IuC",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "qY0ctPbgCbprNs",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "M0qvkSmfO6QVwN",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-MULTI-INTENT",
|
||||
"trace_id": "jR-0DTPMspYzdI",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "AKIvw7axIJsT4d",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.009Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-bXmaw8PKnI-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "LtnlOO3gmIfx8m",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-bXmaw8PKnI
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:23.009Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-eX77Q98Vaz",
|
||||
"run_timestamp": "2026-03-25T20:41:08.913Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.913Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-eX77Q98Vaz-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "4KzkXxobRWVgzp",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.913Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-eX77Q98Vaz-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "iYz_duZbPCRGA3",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-eX77Q98Vaz
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:08.913Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-fnpU4_zMJ_",
|
||||
"run_timestamp": "2026-03-25T20:09:39.795Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "XwIf4z-QoyrqOe",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "Ydcfvt7DwqixDl",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "b7G89RzraXTVwb",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "mv2R58pXN5dWwR",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "NTSe5PDeamqNVU",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "nzGANVjF7ZDCLw",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-MULTI-INTENT",
|
||||
"trace_id": "JfKCiYH6n3R3ZG",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "mzXw7YyUFY7rr6",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:39.795Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-fnpU4_zMJ_-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "VHHVjdZvtSH8o0",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-fnpU4_zMJ_
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:39.795Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-hsqoi_EU-9",
|
||||
"run_timestamp": "2026-03-25T20:09:33.211Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "M02VjwT_OLV_ng",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "WuLHK_WINhH_Vf",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "swKbq62H1LVrif",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "DfjN-tcCW2yUkk",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "LhU1IJY_-kxU-R",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "OqSYQ3cNJo_4p9",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-MULTI-INTENT",
|
||||
"trace_id": "AWOhs31ZqNlLWW",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "iDfyOSi3mTwDCG",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.211Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-hsqoi_EU-9-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "qnILg9teMcQ4yk",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-hsqoi_EU-9
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.211Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,715 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-iK-YSOl5SR",
|
||||
"run_timestamp": "2026-03-25T20:07:25.110Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.33,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "CfVKXT83ECZQeR",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "V08xyKA_D7boQl",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "ia0BqWVmY_dqI6",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "mwh757aLe_4NYo",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "Odap7cFYbxnONe",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "TTHuNxz1VCd9J6",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-MULTI-INTENT",
|
||||
"trace_id": "S-1jQtULWwlY_5",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "Qduj_7t-phwOf7",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:07:25.110Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-iK-YSOl5SR-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "czxOqma9A4IG56",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-iK-YSOl5SR
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:07:25.110Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.33 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-iRPNHs3DIw",
|
||||
"run_timestamp": "2026-03-25T20:40:30.125Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "0",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 0.78,
|
||||
"false_confidence_rate": 0.33,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 0,
|
||||
"label": "Not Actionable",
|
||||
"description": "Бухгалтер не получает понятного следующего шага."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 3,
|
||||
"label": "Moderate False Confidence",
|
||||
"description": "Периодически встречается избыточная уверенность."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "B-Gh0oKEKCQZ0a",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": [
|
||||
"false_confidence_risk"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "SWErDdvdDzpjXH",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "fZ6O0-ZiUO5Vks",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "F3HS-CKCoHsCMR",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "V7g-IpwKb9EJIY",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "no_grounded_answer",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "0SNuPa2zJJAzLu",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 5,
|
||||
"accountant_usefulness_score": 0,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-MULTI-INTENT",
|
||||
"trace_id": "D4qDOFFSg-FXD7",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "jCh9pKAUB09vg8",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 4,
|
||||
"accountant_usefulness_score": 0.75,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.125Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-iRPNHs3DIw-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "s8EYZEcSAQ5ACC",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 1,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": []
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-iRPNHs3DIw
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:40:30.125Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 0.78 | 0 (Not Actionable) |
|
||||
| false_confidence_rate | 0.33 | 3 (Moderate False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, accountant_actionability_score, mechanism_specificity_score
|
||||
@@ -0,0 +1,727 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-kzlMTZa2dL",
|
||||
"run_timestamp": "2026-03-25T20:41:08.741Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.67,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0.22,
|
||||
"broad_answer_rate": 0.25,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "ULEIRHNnv4M3cD",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "gGTj-087JZeg65",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "--M5u_26tqCYMI",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "BfcWG3VC-ry8wK",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "factual_with_explanation",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 1,
|
||||
"broad_answer_rate": 1,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high",
|
||||
"false_confidence_risk",
|
||||
"broad_answer_without_degradation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "HiD27PQoLPVanc",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "X7DHCABvEWf0Np",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-MULTI-INTENT",
|
||||
"trace_id": "ZuglYa5-L2pNl4",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "4-PKh2NnxSwcGb",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:41:08.741Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-kzlMTZa2dL-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "rBVPkJBwjRCG6j",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-kzlMTZa2dL
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:41:08.741Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.67 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0.22 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0.25 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-mgu89bxTBO",
|
||||
"run_timestamp": "2026-03-25T20:09:33.278Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.278Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-mgu89bxTBO-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "TRZ_gLO2IGD3Oy",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:09:33.278Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-mgu89bxTBO-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "YYh9DTUZTM8md_",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "route_mismatch_blocked",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Не подтверждены критичные предметные токены запроса: account_06, account_60"
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-mgu89bxTBO
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:09:33.278Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,723 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-oIPy2jOY3I",
|
||||
"run_timestamp": "2026-03-25T20:40:30.263Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.56,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "NB5-RMPLus6uTr",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "tgNCi2oFrgUeOG",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "Y_XAa-ULfd4YwQ",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "TCFVfO72SUPhDE",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "M26El7YLRdOJP4",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "2hfLq6DLhjx8IO",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-MULTI-INTENT",
|
||||
"trace_id": "_SQb9aUtvhlQuh",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "DEEP4BTGsFOjF4",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.263Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-oIPy2jOY3I-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "1G264bpiVaVnZB",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-oIPy2jOY3I
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:40:30.263Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.56 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-r0wCm_wcya",
|
||||
"run_timestamp": "2026-03-25T20:49:23.082Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 2,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 2,
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 5,
|
||||
"label": "Strong Differentiation",
|
||||
"description": "Ответы устойчиво различаются по предмету и механизму."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 2,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 3
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.082Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-r0wCm_wcya-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "s1WeXtme2hc0Th",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:49:23.082Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-r0wCm_wcya-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "1MXCOgekwPphxQ",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 2
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-r0wCm_wcya
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:49:23.082Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 2
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 1 | 5 (Strong Differentiation) |
|
||||
| generic_explanation_rate | 1 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 3 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 2
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 2
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: retrieval_differentiation_rate, false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
@@ -0,0 +1,723 @@
|
||||
{
|
||||
"schema_version": "assistant_stage1_eval_run_v0_1",
|
||||
"run_id": "assistant-stage1-shCdyu4f_K",
|
||||
"run_timestamp": "2026-03-25T20:40:30.506Z",
|
||||
"eval_target": "assistant_stage1",
|
||||
"mode": "single-pass-strict",
|
||||
"use_mock": true,
|
||||
"prompt_version": "normalizer_v2_0_2",
|
||||
"suite_id": "assistant_stage1_canonical",
|
||||
"suite_version": "0.1.0",
|
||||
"suite_schema_version": "assistant_stage1_suite_v0_1",
|
||||
"scenario_count": 9,
|
||||
"case_ids": [
|
||||
"S1-51-WRONG-CLOSE-TYPE",
|
||||
"S1-60-SUPPLIER-TAILS",
|
||||
"S1-97-LIFECYCLE-ANOMALY",
|
||||
"S1-OS-CARD-VS-CHARGES",
|
||||
"S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"S1-PERIOD-CLOSE-IMPACT",
|
||||
"S1-MULTI-INTENT",
|
||||
"S1-TRANSLIT-QUERY",
|
||||
"S1-FOLLOWUP-INVESTIGATION"
|
||||
],
|
||||
"cases_total": 9,
|
||||
"feature_profile_snapshot": {
|
||||
"FEATURE_ASSISTANT_ACCOUNTANT_EVAL_V1": true,
|
||||
"FEATURE_ASSISTANT_ANSWER_POLICY_V11": "1",
|
||||
"FEATURE_ASSISTANT_BROAD_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_MIN_EVIDENCE_GATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_ANTI_GENERIC_RANKING_GUARD_V1": "1",
|
||||
"FEATURE_ASSISTANT_INVESTIGATION_STATE_V1": "1",
|
||||
"FEATURE_ASSISTANT_STATE_FOLLOWUP_BINDING_V1": "1"
|
||||
},
|
||||
"code_version": {
|
||||
"git_commit": null,
|
||||
"build_marker": "0.1.0"
|
||||
},
|
||||
"metrics": {
|
||||
"raw": {
|
||||
"retrieval_differentiation_rate": 0.56,
|
||||
"generic_explanation_rate": 0.78,
|
||||
"accountant_actionability_score": 2.67,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"denominators": {
|
||||
"cases_total": 9,
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
}
|
||||
},
|
||||
"rubric_bands": {
|
||||
"retrieval_differentiation_rate": {
|
||||
"score": 3,
|
||||
"label": "Partial Differentiation",
|
||||
"description": "Различия есть, но по механизмам недостаточно стабильны."
|
||||
},
|
||||
"generic_explanation_rate": {
|
||||
"score": 0,
|
||||
"label": "Mostly Generic",
|
||||
"description": "Преобладают общие объяснения без локальной опоры."
|
||||
},
|
||||
"accountant_actionability_score": {
|
||||
"score": 3,
|
||||
"label": "Partially Actionable",
|
||||
"description": "Следующий шаг есть, но недостаточно конкретен."
|
||||
},
|
||||
"false_confidence_rate": {
|
||||
"score": 5,
|
||||
"label": "Low False Confidence",
|
||||
"description": "Неопределенность обозначается честно и вовремя."
|
||||
},
|
||||
"broad_answer_rate": {
|
||||
"score": 5,
|
||||
"label": "Controlled",
|
||||
"description": "Broad-ответы редки и сопровождаются корректным сужением."
|
||||
},
|
||||
"mechanism_specificity_score": {
|
||||
"score": 0,
|
||||
"label": "No Mechanism",
|
||||
"description": "Есть только лейблы без механики поломки."
|
||||
},
|
||||
"followup_context_retention_score": {
|
||||
"score": 3,
|
||||
"label": "Context Partial",
|
||||
"description": "Фокус удерживается частично, с дрейфом."
|
||||
}
|
||||
},
|
||||
"subsets": {
|
||||
"broad_cases_total": 8,
|
||||
"followup_cases_total": 1
|
||||
},
|
||||
"budget": {
|
||||
"requests_total": 10
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-51-WRONG-CLOSE-TYPE",
|
||||
"scenario_tag": "51_wrong_closure_type",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-51-WRONG-CLOSE-TYPE",
|
||||
"trace_id": "3rN4CRr_wuXZdc",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "low",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 2,
|
||||
"genericness_score": 1,
|
||||
"accountant_usefulness_score": 2.25,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"weak_source_mapping"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|2020-06-05t12%3A00%3A01",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3Ab2c51d7c54e8e57f54adff3c8aac626509753960|2020-06-23t13%3A54%3A25",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6_recordtype|cmp%3A3e1b4a2f35dad039509c11b74ab38a4614cec9ab|2020-06-08t12%3A13%3A18",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3Ad8cd5a322846d230b68bf26e238469ad65ed037c|2020-06-22t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|2020-06-04t00%3A00%3A00",
|
||||
"evidence_source_ref_v1|unknown|documentjournal_%D0%B1%D0%B0%D0%BD%D0%BA%D0%BE%D0%B2%D1%81%D0%BA%D0%B8%D0%B5%D0%B2%D1%8B%D0%BF%D0%B8%D1%81%D0%BA%D0%B8|bdd1c402-a626-11ea-a2d8-00155d012600|2020-06-04t08%3A46%3A32",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A31ab37b0cbf36a4def1fdf208652ee97b3856493|2020-06-08t12%3A13%3A19",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|2020-06-30t12%3A00%3A00"
|
||||
],
|
||||
"routes": [
|
||||
"hybrid_store_plus_live"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": null,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:weak_source_mapping",
|
||||
"mechanism_unresolved",
|
||||
"Source mapping is weak for part of the evidence.",
|
||||
"Выборка ограничена semantic retrieval profile.",
|
||||
"Рсточник данных — snapshot 2020 (read-only), РЅРµ live контур 1РЎ."
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-60-SUPPLIER-TAILS",
|
||||
"scenario_tag": "60_supplier_tails",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-60-SUPPLIER-TAILS",
|
||||
"trace_id": "9QJLShsRGYo2TI",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-97-LIFECYCLE-ANOMALY",
|
||||
"scenario_tag": "97_lifecycle_anomaly",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-97-LIFECYCLE-ANOMALY",
|
||||
"trace_id": "_FTITsFlSBuB7z",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-OS-CARD-VS-CHARGES",
|
||||
"scenario_tag": "os_card_vs_charges",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-OS-CARD-VS-CHARGES",
|
||||
"trace_id": "vlGDaNOMRtPEXH",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"scenario_tag": "vat_cross_domain_contradiction",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-VAT-CROSS-DOMAIN-CONTRADICTION",
|
||||
"trace_id": "b-AnjTC5e9C5V1",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "clarification_requested",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 2,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "clarification_required",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 0,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки.",
|
||||
"Часть контекстных токенов не подтверждена напрямую: document"
|
||||
],
|
||||
"notes": []
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-PERIOD-CLOSE-IMPACT",
|
||||
"scenario_tag": "period_close_impact",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-PERIOD-CLOSE-IMPACT",
|
||||
"trace_id": "O1LalkRlr6MPS1",
|
||||
"question_type": "broad",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 0,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 0.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "out_of_scope",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": null,
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [],
|
||||
"routes": [
|
||||
"no_route"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 0,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"mechanism_unresolved",
|
||||
"Фрагмент требует уточнения или отсутствует поддерживаемый маршрут.",
|
||||
"Ни одно требование не получило подтвержденного покрытия.",
|
||||
"Нет explainable-сигналов why_included в результатах выборки."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-MULTI-INTENT",
|
||||
"scenario_tag": "multi_intent_question",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-MULTI-INTENT",
|
||||
"trace_id": "6vT7bROIiXpk0j",
|
||||
"question_type": "multi_intent",
|
||||
"broadness_level": "high",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-TRANSLIT-QUERY",
|
||||
"scenario_tag": "translit_query",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-TRANSLIT-QUERY",
|
||||
"trace_id": "ypdQKIxKC9BZbN",
|
||||
"question_type": "direct",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 1,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "weak",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A0a6db56eceac8b185af14b631f6fdf4ab322a291|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%BF%D1%80%D0%B5%D0%B4%D1%8A%D1%8F%D0%B2%D0%BB%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9_recordtype|cmp%3A949e1857e91f167c7caaf0a8f3e3c92db81320d2|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B2%D0%BA%D0%BB%D1%8E%D1%87%D0%B5%D0%BD%D0%BD%D1%8B%D0%B9%D0%B2%D1%81%D1%82%D0%BE%D0%B8%D0%BC%D0%BE%D1%81%D1%82%D1%8C_recordtype|cmp%3A31562c4addff4766b05d673dfb900eb5e53abe30|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A290dab8889920df01e4f0b2f4de7eb3f00e3f4e9|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3Ac44a565eb1c2eb3edeeb3296b0e273d086da02f6|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A6c3846ef170a4031368dcc7b3186ef1aa445f187|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A54593e20ba3ab50efdb118b57d62dfceed97ec04|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A50b747880bcd7aafc708326b92f37d69496b4c68|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A316bc001125a232101215ba6f3fcb88b05b3889f|none",
|
||||
"evidence_source_ref_v1|unknown|accumulationregister_%D0%BD%D0%B4%D1%81%D0%B7%D0%B0%D0%BF%D0%B8%D1%81%D0%B8%D0%BA%D0%BD%D0%B8%D0%B3%D0%B8%D0%BF%D0%BE%D0%BA%D1%83%D0%BF%D0%BE%D0%BA_recordtype|cmp%3A87dafcf0bc010c42b458523937495d924616d69c|none"
|
||||
],
|
||||
"routes": [
|
||||
"store_feature_risk"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 0,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": null
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Риск-факторы определяются эвристикой, а не полным набором бизнес-правил 1С.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
},
|
||||
{
|
||||
"schema_version": "assistant_eval_record_v0_1",
|
||||
"created_at": "2026-03-25T20:40:30.505Z",
|
||||
"case_id": "S1-FOLLOWUP-INVESTIGATION",
|
||||
"scenario_tag": "followup_investigation",
|
||||
"session_id": "assistant-stage1-shCdyu4f_K-S1-FOLLOWUP-INVESTIGATION",
|
||||
"trace_id": "1yeLeAYlD_ItRm",
|
||||
"question_type": "followup",
|
||||
"broadness_level": "medium",
|
||||
"narrowing_result": "not_required",
|
||||
"evidence_quality_score": 1,
|
||||
"genericness_score": 3,
|
||||
"accountant_usefulness_score": 1.5,
|
||||
"accountant_metrics": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"raw_signals": {
|
||||
"final_reply_type": "partial_coverage",
|
||||
"turn_count": 2,
|
||||
"broad_query_detected": false,
|
||||
"broad_result_flag": false,
|
||||
"narrowing_strength": "strong",
|
||||
"minimum_evidence_failed": false,
|
||||
"degraded_to": null,
|
||||
"evidence_confidence": "low",
|
||||
"limitation_reason_codes": [
|
||||
"missing_mechanism"
|
||||
],
|
||||
"mechanism_status": "unresolved",
|
||||
"source_refs": [
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A1|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A2|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A3|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A4|none",
|
||||
"evidence_source_ref_v1|unknown|unknown_entity|batch_refresh_then_store%3Af1%3A5|none"
|
||||
],
|
||||
"routes": [
|
||||
"batch_refresh_then_store"
|
||||
],
|
||||
"followup_state_applied": false
|
||||
},
|
||||
"metric_subscores": {
|
||||
"retrieval_differentiation_rate": 1,
|
||||
"generic_explanation_rate": 1,
|
||||
"accountant_actionability_score": 3,
|
||||
"false_confidence_rate": 0,
|
||||
"broad_answer_rate": 0,
|
||||
"mechanism_specificity_score": 0,
|
||||
"followup_context_retention_score": 3
|
||||
},
|
||||
"limitations": [
|
||||
"limitation_reason:missing_mechanism",
|
||||
"mechanism_unresolved",
|
||||
"Mechanism is unresolved for part of the evidence.",
|
||||
"Ранжирование по объему не всегда эквивалентно бизнес-риску.",
|
||||
"Рчет-токены не подтверждены напрямую (account_60), но есть релевантная опора для ограниченного вывода."
|
||||
],
|
||||
"notes": [
|
||||
"genericness_high"
|
||||
]
|
||||
}
|
||||
],
|
||||
"scenario_summary": {
|
||||
"improved_or_strong": 0,
|
||||
"unchanged_or_mixed": 0,
|
||||
"weak_or_regressed": 9
|
||||
},
|
||||
"improvement_hints": {
|
||||
"strongest_signals": "false_confidence_rate, broad_answer_rate",
|
||||
"weakest_signals": "generic_explanation_rate, mechanism_specificity_score"
|
||||
},
|
||||
"known_limitations": [
|
||||
"Snapshot-only retrieval contour remains (no live verification core in Stage 1).",
|
||||
"Metric mapping for genericness/false confidence is heuristic by design.",
|
||||
"Stage 1 eval excludes Stage 2+ metrics (problem-unit/lifecycle/graph/investigation engine)."
|
||||
],
|
||||
"report_title": "Assistant Stage 1 Eval Run"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Assistant Stage 1 Eval Run
|
||||
|
||||
- run_id: assistant-stage1-shCdyu4f_K
|
||||
- eval_target: assistant_stage1
|
||||
- run_timestamp: 2026-03-25T20:40:30.506Z
|
||||
- suite_id: assistant_stage1_canonical
|
||||
- suite_version: 0.1.0
|
||||
- cases_total: 9
|
||||
|
||||
## Raw Metrics and Rubric Bands
|
||||
|
||||
| Metric | Raw | Rubric band |
|
||||
|---|---:|---|
|
||||
| retrieval_differentiation_rate | 0.56 | 3 (Partial Differentiation) |
|
||||
| generic_explanation_rate | 0.78 | 0 (Mostly Generic) |
|
||||
| accountant_actionability_score | 2.67 | 3 (Partially Actionable) |
|
||||
| false_confidence_rate | 0 | 5 (Low False Confidence) |
|
||||
| broad_answer_rate | 0 | 5 (Controlled) |
|
||||
| mechanism_specificity_score | 0 | 0 (No Mechanism) |
|
||||
| followup_context_retention_score | 3 | 3 (Context Partial) |
|
||||
|
||||
## Subsets
|
||||
|
||||
- broad_cases_total: 8
|
||||
- followup_cases_total: 1
|
||||
|
||||
## Scenario Summary
|
||||
|
||||
- improved_or_strong: 0
|
||||
- unchanged_or_mixed: 0
|
||||
- weak_or_regressed: 9
|
||||
|
||||
## Improvement Hints
|
||||
|
||||
- strongest_signals: false_confidence_rate, broad_answer_rate
|
||||
- weakest_signals: generic_explanation_rate, mechanism_specificity_score
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user