Связать GUI-ревью с issue catalog и detector flywheel
This commit is contained in:
@@ -201,6 +201,76 @@
|
||||
"check": {
|
||||
"target_status": "needs_route_enablement"
|
||||
}
|
||||
},
|
||||
"backend_error_response_signal": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "automatic",
|
||||
"description": "GUI/stage run review found a backend_error user-facing response.",
|
||||
"issue_codes": ["backend_error_response"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "backend_error_response"
|
||||
}
|
||||
},
|
||||
"top_year_direct_answer_missing": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "automatic",
|
||||
"description": "GUI/stage run review found a top-year ranking answer without direct winning-year surface.",
|
||||
"issue_codes": ["year_ranking_direct_answer_missing"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "year_ranking_direct_answer_missing"
|
||||
}
|
||||
},
|
||||
"vat_purchase_date_anchor_missing": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "automatic",
|
||||
"description": "GUI/stage run review found a VAT purchase-date question whose answer did not explain the purchase-date anchor.",
|
||||
"issue_codes": ["purchase_date_vat_anchor_ambiguous"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "purchase_date_vat_anchor_ambiguous"
|
||||
}
|
||||
},
|
||||
"counterparty_value_flow_profit_mismatch": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "automatic",
|
||||
"description": "GUI/stage run review found a counterparty received/paid/net-flow question answered as company profit.",
|
||||
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "counterparty_value_flow_misrouted_to_company_profit"
|
||||
}
|
||||
},
|
||||
"answer_layering_noise_signal": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "semi_automatic",
|
||||
"description": "GUI/stage run review found answer layering noise before the business conclusion.",
|
||||
"issue_codes": ["answer_layering_noise"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "answer_layering_noise"
|
||||
}
|
||||
},
|
||||
"business_answer_too_verbose_signal": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "semi_automatic",
|
||||
"description": "GUI/stage run review found a direct business answer that is too verbose for practical use.",
|
||||
"issue_codes": ["business_answer_too_verbose"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "business_answer_too_verbose"
|
||||
}
|
||||
},
|
||||
"bank_counterparty_role_boundary_missing": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "semi_automatic",
|
||||
"description": "GUI/stage run review found a bank/financial counterparty presented as an ordinary business partner without a role boundary.",
|
||||
"issue_codes": ["bank_counterparty_misclassified_as_business_partner"],
|
||||
"inputs": ["run_review.json", "conversation_pairs.json"],
|
||||
"check": {
|
||||
"target_status": "bank_counterparty_misclassified_as_business_partner"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -257,6 +257,212 @@
|
||||
"route_candidate_pack",
|
||||
"accepted_smoke_pack"
|
||||
]
|
||||
},
|
||||
"backend_error_response": {
|
||||
"severity": "P0",
|
||||
"business_meaning": "Пользовательский ответ сорвался во внутреннюю runtime/LLM ошибку вместо полезного бизнес-ответа или честной границы.",
|
||||
"root_layers": ["runtime_error", "business_utility"],
|
||||
"detectors": ["backend_error_response_signal"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/assistantLivingChatRuntimeAdapter.ts",
|
||||
"llm_normalizer/backend/src/services/assistantLivingChatHandlerRuntimeAdapter.ts",
|
||||
"llm_normalizer/backend/src/services/assistantService.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"fake success fallback",
|
||||
"business routing rewrites",
|
||||
"MCP protocol rewrite"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_gui_turn",
|
||||
"same_saved_session_replay",
|
||||
"big_gui_pack",
|
||||
"accepted_smoke_pack"
|
||||
],
|
||||
"acceptance": {
|
||||
"must_have": [
|
||||
"no backend_error reply_type",
|
||||
"user-facing answer or honest boundary",
|
||||
"trace remains diagnosable"
|
||||
],
|
||||
"must_not_have": [
|
||||
"internal error text",
|
||||
"empty assistant answer",
|
||||
"masked fake business fact"
|
||||
]
|
||||
}
|
||||
},
|
||||
"year_ranking_direct_answer_missing": {
|
||||
"severity": "P0",
|
||||
"business_meaning": "Вопрос про самый доходный/лучший год не получает прямой годовой ranking-ответ в первой строке.",
|
||||
"root_layers": ["answer_shape_mismatch", "business_utility", "ranking_answer_surface"],
|
||||
"detectors": ["top_year_direct_answer_missing"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryRuntimeBridge.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryTurnInputAdapter.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"fake revenue totals",
|
||||
"global business overview rewrite",
|
||||
"silent route masking"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_top_year_turn",
|
||||
"business_overview_counterparty_pack",
|
||||
"big_gui_pack",
|
||||
"accepted_smoke_pack"
|
||||
],
|
||||
"acceptance": {
|
||||
"must_have": [
|
||||
"direct winning year in first answer line",
|
||||
"basis of ranking",
|
||||
"amount or honest limitation",
|
||||
"profit-vs-cashflow limitation if needed"
|
||||
],
|
||||
"must_not_have": [
|
||||
"generic business overview first",
|
||||
"route ids",
|
||||
"unqualified чистая прибыль claim"
|
||||
]
|
||||
}
|
||||
},
|
||||
"purchase_date_vat_anchor_ambiguous": {
|
||||
"severity": "P1",
|
||||
"business_meaning": "Ответ по НДС на дату покупки считает налоговый период, но не объясняет, как дата покупки стала якорем периода.",
|
||||
"root_layers": ["domain_anchor_gap", "field_mapping_gap", "answer_surface"],
|
||||
"detectors": ["vat_purchase_date_anchor_missing"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/address_runtime/composeStage.ts",
|
||||
"llm_normalizer/backend/src/services/addressFilterExtractor.ts",
|
||||
"llm_normalizer/backend/src/services/assistantTransitionPolicy.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"fake purchase date",
|
||||
"VAT route rewrite without evidence",
|
||||
"MCP protocol rewrite"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_vat_purchase_date_turn",
|
||||
"vat_tax_period_pack",
|
||||
"selected_object_followup_pack",
|
||||
"accepted_smoke_pack"
|
||||
],
|
||||
"acceptance": {
|
||||
"must_have": [
|
||||
"purchase date anchor explanation",
|
||||
"tax period window",
|
||||
"sales/purchase book basis",
|
||||
"honest unknown if exact purchase date is inferred"
|
||||
],
|
||||
"must_not_have": [
|
||||
"unexplained period jump",
|
||||
"fake document source",
|
||||
"forecast presented as confirmed declaration"
|
||||
]
|
||||
}
|
||||
},
|
||||
"counterparty_value_flow_misrouted_to_company_profit": {
|
||||
"severity": "P0",
|
||||
"business_meaning": "Контрагентский вопрос про получили/заплатили/нетто отвечен company-level прибылью или 90/91/99 вместо контрагентского денежного потока.",
|
||||
"root_layers": ["followup_action_resolution_gap", "answer_shape_mismatch", "domain_scope_leak"],
|
||||
"detectors": ["counterparty_value_flow_profit_mismatch"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryRuntimeBridge.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryTurnInputAdapter.ts",
|
||||
"llm_normalizer/backend/src/services/assistantTransitionPolicy.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"fake counterparty totals",
|
||||
"company profit fallback",
|
||||
"global orchestration rewrite"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_counterparty_value_flow_turn",
|
||||
"business_overview_counterparty_pack",
|
||||
"cross_scope_followup_pack",
|
||||
"accepted_smoke_pack"
|
||||
],
|
||||
"acceptance": {
|
||||
"must_have": [
|
||||
"counterparty scope",
|
||||
"received amount or honest unknown",
|
||||
"paid amount or honest unknown",
|
||||
"net amount or honest unknown",
|
||||
"company-profit boundary"
|
||||
],
|
||||
"must_not_have": [
|
||||
"90/91/99 as direct counterparty answer",
|
||||
"company profit as counterparty net",
|
||||
"wrong selected entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
"answer_layering_noise": {
|
||||
"severity": "P1",
|
||||
"business_meaning": "Ответ открывается лишним слоем рассуждений/рамок вместо короткого прямого вывода.",
|
||||
"root_layers": ["answer_surface", "business_utility"],
|
||||
"detectors": ["answer_layering_noise_signal"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/address_runtime/composeStage.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"fake evidence",
|
||||
"route masking"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_gui_turn",
|
||||
"direct_answer_surface_pack",
|
||||
"accepted_smoke_pack"
|
||||
]
|
||||
},
|
||||
"business_answer_too_verbose": {
|
||||
"severity": "P1",
|
||||
"business_meaning": "Ответ слишком длинный для прямого бизнес-вопроса и снижает практическую полезность.",
|
||||
"root_layers": ["answer_surface", "business_utility"],
|
||||
"detectors": ["business_answer_too_verbose_signal"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/address_runtime/composeStage.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"evidence deletion from artifacts",
|
||||
"business fact removal"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_gui_turn",
|
||||
"direct_answer_surface_pack",
|
||||
"accepted_smoke_pack"
|
||||
]
|
||||
},
|
||||
"bank_counterparty_misclassified_as_business_partner": {
|
||||
"severity": "P1",
|
||||
"business_meaning": "Банк/финансовая организация ошибочно подана как обычный клиент/поставщик без роли, назначения платежа и договорного контекста.",
|
||||
"root_layers": ["business_semantic_role_gap", "domain_scope"],
|
||||
"detectors": ["bank_counterparty_role_boundary_missing"],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/address_runtime/composeStage.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"scripts/review_assistant_stage1_run.py"
|
||||
],
|
||||
"forbidden_patch_targets": [
|
||||
"hard-coded bank blacklist as final truth",
|
||||
"fake operation purpose",
|
||||
"global routing rewrite"
|
||||
],
|
||||
"rerun_matrix": [
|
||||
"failed_bank_counterparty_turn",
|
||||
"counterparty_value_pack",
|
||||
"accepted_smoke_pack"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user