Укрепить агентный loop Phase107
This commit is contained in:
parent
b3f2e405bb
commit
41463c9965
|
|
@ -1,10 +1,10 @@
|
||||||
{
|
{
|
||||||
"schema_version": "domain_scenario_pack_v1",
|
"schema_version": "domain_scenario_pack_v1",
|
||||||
"pack_id": "active_phase106_profit_cashflow_margin_next_step_20260615",
|
"pack_id": "active_phase107_mixed_planner_brain_dogfood_20260615",
|
||||||
"domain": "address_phase106_profit_cashflow_margin_next_step",
|
"domain": "address_phase107_mixed_planner_brain_dogfood",
|
||||||
"runtime_domain": "business_overview_profit_cashflow_margin_boundary",
|
"runtime_domain": "planner_autonomy_mixed_boundary",
|
||||||
"title": "Profit, cashflow, margin and next-check boundary",
|
"title": "Mixed planner-brain dogfood replay",
|
||||||
"description": "Active slice for the autonomous agent loop: colloquial earnings wording must be answered as checked cashflow/accounting evidence with honest profit boundaries, then pivot safely into товарная маржа and concrete next checks.",
|
"description": "Active slice for dogfooding the autonomous agent loop on a compact mixed planner path: living chat guard, counterparty alias grounding, follow-up money flow, net, documents, movement evidence, broad business evaluation, and off-domain recovery.",
|
||||||
"status": "active",
|
"status": "active",
|
||||||
"source_of_truth_policy": {
|
"source_of_truth_policy": {
|
||||||
"purpose": "single mutable domain source for the current orchestration target",
|
"purpose": "single mutable domain source for the current orchestration target",
|
||||||
|
|
@ -21,182 +21,349 @@
|
||||||
},
|
},
|
||||||
"analysis_context": {
|
"analysis_context": {
|
||||||
"as_of_date": "2026-06-15",
|
"as_of_date": "2026-06-15",
|
||||||
"source": "phase106_after_phase104_role_tail_autorun_persistence",
|
"source": "phase107_after_phase106_profit_cashflow_margin_next_step",
|
||||||
"known_previous_anchors": [
|
"known_previous_anchors": [
|
||||||
"hm_business_overview_cashflow_limits_20260602_p4",
|
"phase83_planner_brain_alignment_mix_20260603_live9",
|
||||||
"hm_business_profile_evaluation_limits_p3",
|
"phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop",
|
||||||
"agent_margin_profitability_reliability_20260615_p12_detector_scope",
|
"phase106_profit_cashflow_margin_next_step_20260615_p03_loop"
|
||||||
"phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop"
|
],
|
||||||
]
|
"why_now": "Phase83 passed in the legacy truth-harness format; this pack replays the highest-value mixed planner slice through the current autonomous pack-loop."
|
||||||
},
|
},
|
||||||
"target_score": 88,
|
"target_score": 88,
|
||||||
"bindings": {
|
"bindings": {
|
||||||
"main_organization": "ООО Альтернатива Плюс",
|
"main_organization": "ООО Альтернатива Плюс",
|
||||||
"control_year": "2020"
|
"control_year": "2020",
|
||||||
|
"svk_counterparty": "Группа СВК"
|
||||||
},
|
},
|
||||||
"issue_codes_under_test": [
|
"issue_codes_under_test": [
|
||||||
"cashflow_misreported_as_clean_profit",
|
"technical_garbage_in_answer",
|
||||||
"margin_domain_leak_accounting_route",
|
"business_direct_answer_missing",
|
||||||
"business_next_step_missing",
|
"business_next_step_missing",
|
||||||
"technical_garbage_in_answer"
|
"counterparty_value_flow_misrouted_to_company_profit",
|
||||||
|
"business_utility_gap"
|
||||||
],
|
],
|
||||||
"detectors_under_test": [
|
"detectors_under_test": [
|
||||||
"runtime_tokens_in_user_answer",
|
"runtime_tokens_in_user_answer",
|
||||||
"capability_ids_in_user_answer",
|
"capability_ids_in_user_answer",
|
||||||
|
"first_line_not_direct_answer",
|
||||||
|
"top_level_scaffold_before_answer",
|
||||||
"limited_answer_without_next_action",
|
"limited_answer_without_next_action",
|
||||||
"margin_domain_leak_accounting_route",
|
"counterparty_value_flow_required_surface",
|
||||||
"margin_required_fields_missing",
|
"counterparty_value_flow_profit_accounts_forbidden"
|
||||||
"margin_next_action_missing"
|
|
||||||
],
|
],
|
||||||
"agent_audit_expectations": {
|
"agent_audit_expectations": {
|
||||||
"semantic_answer_review_first": true,
|
"semantic_answer_review_first": true,
|
||||||
"direct_answer_first": true,
|
"direct_answer_first": true,
|
||||||
"business_utility_required": true,
|
"business_utility_required": true,
|
||||||
"cashflow_not_clean_profit": true,
|
"selected_counterparty_memory_required": true,
|
||||||
"margin_domain_purity_required": true,
|
"date_carryover_required": true,
|
||||||
"next_action_required_when_limited": true,
|
"documents_and_movements_are_evidence_drilldowns": true,
|
||||||
|
"off_domain_living_chat_must_not_replay_business_context": true,
|
||||||
"technical_garbage_forbidden": true,
|
"technical_garbage_forbidden": true,
|
||||||
"do_not_accept_if": [
|
"do_not_accept_if": [
|
||||||
"colloquial earnings wording is answered as confirmed clean profit without evidence",
|
"a follow-up over 'нему' loses the selected counterparty",
|
||||||
"cashflow/net operating flow is silently presented as accounting profit or margin",
|
"a payout or net follow-up loses the 2020 period",
|
||||||
"margin follow-up leaks into bank payments, fixed assets, amortization, or generic accounting route",
|
"a counterparty received/paid/net question is answered as company profit",
|
||||||
"next-check answer is generic advice without concrete 1C evidence contours",
|
"document or movement follow-up answers with a fresh clarification instead of reusing the selected counterparty",
|
||||||
"final answer leaks route ids, capability ids, runtime enums, or debug payload"
|
"broad business evaluation leaks route ids, capability ids, runtime enums, or debug payload",
|
||||||
|
"off-domain living chat is hijacked by stale 1C business context"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"scenarios": [
|
"scenarios": [
|
||||||
{
|
{
|
||||||
"scenario_id": "profit_cashflow_margin_next_step_boundary",
|
"scenario_id": "mixed_planner_counterparty_evidence_and_living_guard",
|
||||||
"title": "Colloquial earnings, clean-profit boundary, margin pivot, and next checks",
|
"title": "Counterparty memory, evidence drilldowns, business overview, and off-domain guard",
|
||||||
"description": "Validate that the assistant handles colloquial earnings wording, refuses to overclaim clean profit, pivots into товарная маржа without wrong-domain leakage, and proposes concrete next 1C checks.",
|
"description": "Validate a compact but cross-cutting planner path: chat sanity, loose counterparty alias grounding, follow-up money flow and net, documents/movements evidence, explicit broad company evaluation, and recovery into ordinary living chat.",
|
||||||
"steps": [
|
"steps": [
|
||||||
{
|
{
|
||||||
"step_id": "step_01_choose_company_scope",
|
"step_id": "step_01_human_smalltalk_sanity",
|
||||||
"title": "Bare organization choice binds active company scope",
|
"title": "Living chat remains human and does not expose discovery internals",
|
||||||
"node_role": "root",
|
"node_role": "root",
|
||||||
"question": "{{bindings.main_organization}}",
|
"question": "привет, ты на связи?",
|
||||||
"allowed_reply_types": [
|
"allowed_reply_types": [
|
||||||
"factual_with_explanation",
|
"factual_with_explanation",
|
||||||
"partial_coverage"
|
"partial_coverage"
|
||||||
],
|
],
|
||||||
"semantic_tags": [
|
"semantic_tags": [
|
||||||
"bare_org_scope",
|
"human_answer",
|
||||||
"company_scope"
|
"mcp_discovery_gate_sanity",
|
||||||
],
|
"meta_smalltalk"
|
||||||
"required_answer_shape": "direct_answer_first",
|
|
||||||
"required_answer_patterns_all": [
|
|
||||||
"(?i)фиксир|рабоч.*организац|контур",
|
|
||||||
"(?i)альтернатива"
|
|
||||||
],
|
|
||||||
"forbidden_answer_patterns": [
|
|
||||||
"(?i)не могу определить",
|
|
||||||
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"step_id": "step_02_colloquial_earned_money_2020",
|
|
||||||
"title": "Colloquial earned-money wording stays bounded",
|
|
||||||
"question": "скока денег альтернатива заработала за 20 год?",
|
|
||||||
"depends_on": [
|
|
||||||
"step_01_choose_company_scope"
|
|
||||||
],
|
|
||||||
"semantic_tags": [
|
|
||||||
"business_overview",
|
|
||||||
"colloquial_cashflow",
|
|
||||||
"cashflow_not_profit"
|
|
||||||
],
|
|
||||||
"required_answer_shape": "direct_answer_first",
|
|
||||||
"required_answer_patterns_all": [
|
|
||||||
"(?i)2020|20\\s*год",
|
|
||||||
"(?i)поступ|входящ|денег|заработ",
|
|
||||||
"(?i)не.*чист.*прибыл|не.*прибыл|прибыл.*не подтвержд|это.*не.*прибыл"
|
|
||||||
],
|
|
||||||
"forbidden_answer_patterns": [
|
|
||||||
"(?i)^\\s*Коротко:",
|
|
||||||
"(?i)это\\s+чистая\\s+прибыль",
|
|
||||||
"(?i)точно.*прибыл",
|
|
||||||
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
|
||||||
],
|
|
||||||
"criticality": "critical"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"step_id": "step_03_clean_profit_followup_boundary",
|
|
||||||
"title": "Clean-profit follow-up must not convert cashflow into profit",
|
|
||||||
"question": "а это чистая прибыль?",
|
|
||||||
"depends_on": [
|
|
||||||
"step_02_colloquial_earned_money_2020"
|
|
||||||
],
|
|
||||||
"semantic_tags": [
|
|
||||||
"profitability_boundary",
|
|
||||||
"cashflow_not_profit",
|
|
||||||
"followup_boundary"
|
|
||||||
],
|
|
||||||
"required_answer_shape": "direct_answer_first",
|
|
||||||
"required_answer_patterns_all": [
|
|
||||||
"(?i)не.*чист.*прибыл|не.*является.*прибыл|не подтвержд.*прибыл|это.*не.*прибыл",
|
|
||||||
"(?i)денежн|поступ|нетто|оборот|cash|себестоим|расход|закрыва"
|
|
||||||
],
|
|
||||||
"forbidden_answer_patterns": [
|
|
||||||
"(?i)^\\s*Коротко:",
|
|
||||||
"(?i)^\\s*да[,\\s]+это.*прибыл",
|
|
||||||
"(?i)точно.*чист.*прибыл",
|
|
||||||
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
|
||||||
],
|
|
||||||
"criticality": "critical"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"step_id": "step_04_margin_pivot_keeps_inventory_domain",
|
|
||||||
"title": "Margin pivot uses inventory margin contour, not payments or fixed assets",
|
|
||||||
"question": "тогда какую прибыль или маржу можно проверить по товарам за 2020?",
|
|
||||||
"depends_on": [
|
|
||||||
"step_03_clean_profit_followup_boundary"
|
|
||||||
],
|
|
||||||
"expected_intents": [
|
|
||||||
"inventory_margin_ranking_for_nomenclature"
|
|
||||||
],
|
|
||||||
"expected_capability": "inventory_inventory_margin_ranking_for_nomenclature",
|
|
||||||
"expected_recipe": "address_inventory_margin_ranking_for_nomenclature_v1",
|
|
||||||
"semantic_tags": [
|
|
||||||
"margin_profitability",
|
|
||||||
"inventory",
|
|
||||||
"wrong_domain_trap",
|
|
||||||
"cashflow_to_margin_pivot"
|
|
||||||
],
|
],
|
||||||
"required_answer_shape": "direct_answer_first",
|
"required_answer_shape": "direct_answer_first",
|
||||||
"required_answer_patterns_any": [
|
"required_answer_patterns_any": [
|
||||||
"(?i)марж|валов|выруч|себестоим|номенклатур|товар|не могу подтвердить|не хватает"
|
"(?i)привет|на связи|готов|помочь"
|
||||||
],
|
],
|
||||||
"forbidden_answer_patterns": [
|
"forbidden_answer_patterns": [
|
||||||
"(?i)амортизац|основн(ые|ых)? средств|объект ОС|банк|оплат[аы]|payment_document|settlement",
|
"(?i)mcp|runtime_|query_documents|primitive|route_candidate|capability_id|snapshot_items|answer_object"
|
||||||
|
],
|
||||||
|
"criticality": "info"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_02_resolve_counterparty_alias",
|
||||||
|
"title": "Entity resolution grounds the checked 1C counterparty from a loose alias",
|
||||||
|
"question": "найди в 1С контрагента СВК",
|
||||||
|
"depends_on": [
|
||||||
|
"step_01_human_smalltalk_sanity"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual",
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"entity_resolution",
|
||||||
|
"alias_grounding",
|
||||||
|
"followup_anchor",
|
||||||
|
"planner_catalog_alignment"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)свк",
|
||||||
|
"(?i)контрагент"
|
||||||
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)группа\\s+свк",
|
||||||
|
"(?i)каталог|найден|наиболее вероятн"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)получили|заплатили|нетто|оборот|выручк|сумм(а|ы)",
|
||||||
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
],
|
],
|
||||||
"criticality": "critical"
|
"criticality": "critical"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step_id": "step_05_next_checks_are_concrete_1c_contours",
|
"step_id": "step_03_incoming_by_resolved_entity",
|
||||||
"title": "Next checks must be concrete 1C evidence contours",
|
"title": "Incoming value-flow follow-up reuses the resolved counterparty anchor",
|
||||||
"question": "что дальше проверить, чтобы понять здоровье бизнеса, без гаданий?",
|
"question": "сколько получили по нему за 2020 год",
|
||||||
"depends_on": [
|
"depends_on": [
|
||||||
"step_04_margin_pivot_keeps_inventory_domain"
|
"step_02_resolve_counterparty_alias"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
],
|
],
|
||||||
"semantic_tags": [
|
"semantic_tags": [
|
||||||
"business_next_step",
|
"entity_resolution",
|
||||||
"evidence_plan",
|
"incoming_value_flow",
|
||||||
"no_generic_advice"
|
"followup_reuse",
|
||||||
|
"date_carryover",
|
||||||
|
"planner_catalog_alignment"
|
||||||
],
|
],
|
||||||
"required_answer_shape": "direct_answer_first",
|
"required_answer_shape": "direct_answer_first",
|
||||||
"required_answer_patterns_all": [
|
"required_answer_patterns_all": [
|
||||||
"(?i)прибыл|марж|себестоим|финрезульт",
|
"(?i)2020",
|
||||||
"(?i)ндс|налог",
|
"(?i)получил|входящ|поступ",
|
||||||
"(?i)дебитор|кредитор|долг|расчет",
|
"(?i)руб"
|
||||||
"(?i)склад|остат|оборач|товар"
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)группа\\s+свк",
|
||||||
|
"(?i)свк"
|
||||||
],
|
],
|
||||||
"forbidden_answer_patterns": [
|
"forbidden_answer_patterns": [
|
||||||
"(?i)^\\s*Коротко:",
|
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
|
||||||
"(?i)просто.*посмотр|общ.*рекоменд|улучшить маркетинг|нанять|стратег",
|
"(?i)чист.*прибыл|90/91/99|финрезульт",
|
||||||
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
],
|
],
|
||||||
"criticality": "important"
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_04_payout_switch_by_resolved_entity",
|
||||||
|
"title": "Outgoing payment follow-up keeps the same grounded counterparty and checked year",
|
||||||
|
"question": "а теперь сколько заплатили?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_03_incoming_by_resolved_entity"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"entity_resolution",
|
||||||
|
"payout_switch",
|
||||||
|
"followup_reuse",
|
||||||
|
"date_carryover",
|
||||||
|
"planner_catalog_alignment"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)2020",
|
||||||
|
"(?i)заплатил|исходящ|списан|платеж|платёж",
|
||||||
|
"(?i)руб"
|
||||||
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)группа\\s+свк",
|
||||||
|
"(?i)свк"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
|
||||||
|
"(?i)чист.*прибыл|90/91/99|финрезульт",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_05_net_after_payout",
|
||||||
|
"title": "Net-flow follow-up reuses the same grounded counterparty and checked year after payout",
|
||||||
|
"question": "а какое нетто?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_04_payout_switch_by_resolved_entity"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"entity_resolution",
|
||||||
|
"net_value_flow",
|
||||||
|
"followup_reuse",
|
||||||
|
"date_carryover",
|
||||||
|
"planner_catalog_alignment"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)2020",
|
||||||
|
"(?i)нетто|сальдо|разниц",
|
||||||
|
"(?i)получ",
|
||||||
|
"(?i)заплат",
|
||||||
|
"(?i)руб"
|
||||||
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"12[ .\\u00a0]?093[ .\\u00a0]?465|12093465",
|
||||||
|
"(?i)группа\\s+свк|свк"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
|
||||||
|
"(?i)чист.*прибыл|90/91/99|финрезульт",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_06_documents_after_net",
|
||||||
|
"title": "Document evidence follow-up keeps the grounded counterparty after the net answer",
|
||||||
|
"question": "а по документам?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_05_net_after_payout"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual",
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"entity_resolution",
|
||||||
|
"document_evidence",
|
||||||
|
"value_flow_pivot",
|
||||||
|
"followup_reuse",
|
||||||
|
"planner_catalog_alignment"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)документ|счет|счёт|накладн|акт"
|
||||||
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)группа\\s+свк",
|
||||||
|
"(?i)свк",
|
||||||
|
"(?i)2020"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
|
||||||
|
"(?i)сколько получили|сколько заплатили|нетто",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_07_movements_after_documents",
|
||||||
|
"title": "Movement evidence follow-up keeps the grounded counterparty after the document answer",
|
||||||
|
"question": "а по движениям?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_06_documents_after_net"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual",
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"entity_resolution",
|
||||||
|
"movement_evidence",
|
||||||
|
"document_pivot",
|
||||||
|
"followup_reuse",
|
||||||
|
"planner_catalog_alignment"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)движени|операц|платеж|платёж|списан|поступ"
|
||||||
|
],
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)группа\\s+свк",
|
||||||
|
"(?i)свк",
|
||||||
|
"(?i)2020"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
|
||||||
|
"(?i)сколько получили|сколько заплатили|нетто",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_08_broad_business_evaluation_explicit_org",
|
||||||
|
"title": "Broad business evaluation stays grounded and bounded after counterparty drilldowns",
|
||||||
|
"question": "Как ты оценишь деятельность компании {{bindings.main_organization}} по данным 1С?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_07_movements_after_documents"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"broad_business_evaluation",
|
||||||
|
"grounded_summary",
|
||||||
|
"context_switch_from_counterparty_to_company"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_all": [
|
||||||
|
"(?i)альтернатива",
|
||||||
|
"(?i)ограниченн|проверенн|не\\s+аудит|не\\s+аудитор",
|
||||||
|
"(?i)1с|подтвержд",
|
||||||
|
"(?i)денежн|долг|ндс|контрагент|операц"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)^\\s*коротко:",
|
||||||
|
"(?i)чистая\\s+прибыль\\s+подтверждена|аудиторское\\s+заключение",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "critical"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"step_id": "step_09_off_domain_living_chat_not_hijacked",
|
||||||
|
"title": "Off-domain living chat is not hijacked by stale business context",
|
||||||
|
"question": "а чем капибара отличается от утки?",
|
||||||
|
"depends_on": [
|
||||||
|
"step_08_broad_business_evaluation_explicit_org"
|
||||||
|
],
|
||||||
|
"allowed_reply_types": [
|
||||||
|
"factual_with_explanation",
|
||||||
|
"partial_coverage"
|
||||||
|
],
|
||||||
|
"semantic_tags": [
|
||||||
|
"off_domain_living_chat",
|
||||||
|
"stale_replay_forbidden",
|
||||||
|
"context_boundary"
|
||||||
|
],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"required_answer_patterns_any": [
|
||||||
|
"(?i)капибар.*утк|утк.*капибар",
|
||||||
|
"(?i)млекопита|птиц|грызун"
|
||||||
|
],
|
||||||
|
"forbidden_answer_patterns": [
|
||||||
|
"(?i)альтернатива|свк|контрагент|1с|поступлен|платеж|платёж|нетто",
|
||||||
|
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
|
||||||
|
],
|
||||||
|
"criticality": "warning"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -218,7 +218,9 @@
|
||||||
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||||
"inputs": ["steps/<step_id>/output.md"],
|
"inputs": ["steps/<step_id>/output.md"],
|
||||||
"check": {
|
"check": {
|
||||||
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
|
"artifact_path_include_patterns": [
|
||||||
|
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_05_net_after_payout)"
|
||||||
|
],
|
||||||
"required_patterns_any": [
|
"required_patterns_any": [
|
||||||
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
|
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
|
||||||
]
|
]
|
||||||
|
|
@ -231,7 +233,9 @@
|
||||||
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||||
"inputs": ["steps/<step_id>/output.md"],
|
"inputs": ["steps/<step_id>/output.md"],
|
||||||
"check": {
|
"check": {
|
||||||
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
|
"artifact_path_include_patterns": [
|
||||||
|
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_03_incoming_by_resolved_entity|step_04_payout_switch_by_resolved_entity|step_05_net_after_payout)"
|
||||||
|
],
|
||||||
"forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"]
|
"forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -18,8 +19,35 @@ DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def path_for_io(path: Path) -> str:
|
||||||
|
if os.name != "nt":
|
||||||
|
return str(path)
|
||||||
|
raw_value = str(path)
|
||||||
|
if raw_value.startswith("\\\\?\\"):
|
||||||
|
return raw_value
|
||||||
|
absolute_path = path if path.is_absolute() else Path.cwd() / path
|
||||||
|
absolute_value = str(absolute_path.absolute())
|
||||||
|
if absolute_value.startswith("\\\\?\\"):
|
||||||
|
return absolute_value
|
||||||
|
if absolute_value.startswith("\\\\"):
|
||||||
|
return "\\\\?\\UNC\\" + absolute_value.lstrip("\\")
|
||||||
|
return "\\\\?\\" + absolute_value
|
||||||
|
|
||||||
|
|
||||||
|
def path_exists(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
return os.path.exists(path_for_io(path))
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(path: Path) -> str:
|
||||||
|
with open(path_for_io(path), "r", encoding="utf-8") as handle:
|
||||||
|
return handle.read()
|
||||||
|
|
||||||
|
|
||||||
def read_json(path: Path) -> Any:
|
def read_json(path: Path) -> Any:
|
||||||
return json.loads(path.read_text(encoding="utf-8"))
|
return json.loads(read_text(path))
|
||||||
|
|
||||||
|
|
||||||
def read_json_object(path: Path) -> dict[str, Any]:
|
def read_json_object(path: Path) -> dict[str, Any]:
|
||||||
|
|
@ -31,8 +59,9 @@ def read_json_object(path: Path) -> dict[str, Any]:
|
||||||
|
|
||||||
|
|
||||||
def write_json(path: Path, payload: Any) -> None:
|
def write_json(path: Path, payload: Any) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
os.makedirs(path_for_io(path.parent), exist_ok=True)
|
||||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
with open(path_for_io(path), "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n")
|
||||||
|
|
||||||
|
|
||||||
def utc_now() -> str:
|
def utc_now() -> str:
|
||||||
|
|
@ -164,7 +193,7 @@ def select_detectors(
|
||||||
|
|
||||||
def read_text_or_empty(path: Path) -> str:
|
def read_text_or_empty(path: Path) -> str:
|
||||||
try:
|
try:
|
||||||
return path.read_text(encoding="utf-8")
|
return read_text(path)
|
||||||
except OSError:
|
except OSError:
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
@ -207,7 +236,7 @@ def first_pattern_search(patterns: list[re.Pattern[str]], text: str) -> re.Match
|
||||||
|
|
||||||
|
|
||||||
def assistant_text_from_turn_path(path: Path | None) -> str:
|
def assistant_text_from_turn_path(path: Path | None) -> str:
|
||||||
if path is None or not path.exists():
|
if path is None or not path_exists(path):
|
||||||
return ""
|
return ""
|
||||||
payload = read_json_object(path)
|
payload = read_json_object(path)
|
||||||
assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {}
|
assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {}
|
||||||
|
|
@ -264,14 +293,21 @@ def output_turn_path(output_path: Path) -> Path | None:
|
||||||
name = output_path.name
|
name = output_path.name
|
||||||
if name == "output.md":
|
if name == "output.md":
|
||||||
candidate = output_path.with_name("turn.json")
|
candidate = output_path.with_name("turn.json")
|
||||||
return candidate if candidate.exists() else None
|
return candidate if path_exists(candidate) else None
|
||||||
if name.endswith("_output.md"):
|
if name.endswith("_output.md"):
|
||||||
prefix = name[: -len("_output.md")]
|
prefix = name[: -len("_output.md")]
|
||||||
candidate = output_path.with_name(f"{prefix}_turn.json")
|
candidate = output_path.with_name(f"{prefix}_turn.json")
|
||||||
return candidate if candidate.exists() else None
|
return candidate if path_exists(candidate) else None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def output_step_state_path(output_path: Path) -> Path | None:
|
||||||
|
if output_path.name != "output.md":
|
||||||
|
return None
|
||||||
|
candidate = output_path.with_name("step_state.json")
|
||||||
|
return candidate if path_exists(candidate) else None
|
||||||
|
|
||||||
|
|
||||||
def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
|
def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
|
||||||
outputs: list[dict[str, Any]] = []
|
outputs: list[dict[str, Any]] = []
|
||||||
seen: set[Path] = set()
|
seen: set[Path] = set()
|
||||||
|
|
@ -292,6 +328,7 @@ def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
|
||||||
"artifact_path": str(path.relative_to(artifact_dir)),
|
"artifact_path": str(path.relative_to(artifact_dir)),
|
||||||
"text": read_text_or_empty(path),
|
"text": read_text_or_empty(path),
|
||||||
"turn_path": turn_path,
|
"turn_path": turn_path,
|
||||||
|
"step_state_path": output_step_state_path(path),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return outputs
|
return outputs
|
||||||
|
|
@ -497,6 +534,52 @@ def evaluate_limited_next_action(
|
||||||
return build_result(detector_name, detector, status, message, evidence=failures)
|
return build_result(detector_name, detector, status, message, evidence=failures)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_answer_text_shape(
|
||||||
|
detector_name: str,
|
||||||
|
detector: dict[str, Any],
|
||||||
|
outputs: list[dict[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not outputs:
|
||||||
|
return build_result(detector_name, detector, "skipped", "no output.md-style artifacts matched detector scope")
|
||||||
|
failures: list[dict[str, Any]] = []
|
||||||
|
reviewed: list[dict[str, Any]] = []
|
||||||
|
unknown: list[dict[str, Any]] = []
|
||||||
|
for output in outputs:
|
||||||
|
step_state_path = output.get("step_state_path")
|
||||||
|
step_state = read_json_object(step_state_path) if isinstance(step_state_path, Path) else {}
|
||||||
|
review = step_state.get("business_first_review") if isinstance(step_state.get("business_first_review"), dict) else {}
|
||||||
|
direct_answer_first_ok = review.get("direct_answer_first_ok")
|
||||||
|
if direct_answer_first_ok is True:
|
||||||
|
reviewed.append({"path": output["repo_path"], "direct_answer_first_ok": True})
|
||||||
|
elif direct_answer_first_ok is False:
|
||||||
|
failures.append({"path": output["repo_path"], "direct_answer_first_ok": False})
|
||||||
|
else:
|
||||||
|
unknown.append({"path": output["repo_path"], "reason": "business_first_review_missing"})
|
||||||
|
if failures:
|
||||||
|
return build_result(
|
||||||
|
detector_name,
|
||||||
|
detector,
|
||||||
|
"fail",
|
||||||
|
"first-line direct answer check failed",
|
||||||
|
evidence=[*failures, *unknown],
|
||||||
|
)
|
||||||
|
if unknown:
|
||||||
|
return build_result(
|
||||||
|
detector_name,
|
||||||
|
detector,
|
||||||
|
"review",
|
||||||
|
"direct-answer shape requires business review",
|
||||||
|
evidence=[*reviewed, *unknown],
|
||||||
|
)
|
||||||
|
return build_result(
|
||||||
|
detector_name,
|
||||||
|
detector,
|
||||||
|
"pass",
|
||||||
|
"business-first step reviews confirm direct-answer-first shape",
|
||||||
|
evidence=reviewed,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def evaluate_trace_guard(
|
def evaluate_trace_guard(
|
||||||
detector_name: str,
|
detector_name: str,
|
||||||
detector: dict[str, Any],
|
detector: dict[str, Any],
|
||||||
|
|
@ -686,7 +769,7 @@ def evaluate_detector(
|
||||||
):
|
):
|
||||||
return evaluate_composite(detector_name, detector, results_by_name)
|
return evaluate_composite(detector_name, detector, results_by_name)
|
||||||
if kind == "answer_text_shape":
|
if kind == "answer_text_shape":
|
||||||
return evaluate_manual_review(detector_name, detector, "direct-answer shape requires business review")
|
return evaluate_answer_text_shape(detector_name, detector, scoped_outputs)
|
||||||
return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}")
|
return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -317,6 +317,7 @@ DEFAULT_INVARIANT_SEVERITY: dict[str, str] = {
|
||||||
"forbidden_recipe_selected": "P0",
|
"forbidden_recipe_selected": "P0",
|
||||||
"focus_object_missing": "P0",
|
"focus_object_missing": "P0",
|
||||||
"wrong_date_scope_state": "P0",
|
"wrong_date_scope_state": "P0",
|
||||||
|
"out_of_window_date_in_answer": "P0",
|
||||||
"direct_answer_missing": "P0",
|
"direct_answer_missing": "P0",
|
||||||
"top_level_noise_present": "P0",
|
"top_level_noise_present": "P0",
|
||||||
"business_direct_answer_missing": "P0",
|
"business_direct_answer_missing": "P0",
|
||||||
|
|
@ -930,6 +931,40 @@ def first_non_empty_lines(text: str, limit: int = 3) -> list[str]:
|
||||||
return output
|
return output
|
||||||
|
|
||||||
|
|
||||||
|
def expected_single_year_from_date_scope(date_scope: Any) -> str | None:
|
||||||
|
if not isinstance(date_scope, dict):
|
||||||
|
return None
|
||||||
|
if str(date_scope.get("scope") or "").strip().lower() == "all_time":
|
||||||
|
return None
|
||||||
|
period_from = normalize_iso_date(date_scope.get("period_from"))
|
||||||
|
period_to = normalize_iso_date(date_scope.get("period_to"))
|
||||||
|
if not period_from or not period_to:
|
||||||
|
return None
|
||||||
|
from_year = period_from[:4]
|
||||||
|
to_year = period_to[:4]
|
||||||
|
if from_year != to_year:
|
||||||
|
return None
|
||||||
|
return from_year if re.fullmatch(r"(?:19|20)\d{2}", from_year) else None
|
||||||
|
|
||||||
|
|
||||||
|
def dated_evidence_years_from_answer(text: Any) -> list[str]:
|
||||||
|
source = repair_text_mojibake(str(text or ""))
|
||||||
|
years: list[str] = []
|
||||||
|
for match in re.finditer(r"\b((?:19|20)\d{2})-\d{2}-\d{2}(?:T|\b)", source):
|
||||||
|
years.append(match.group(1))
|
||||||
|
for match in re.finditer(r"\b\d{1,2}\.\d{1,2}\.((?:19|20)\d{2})\b", source):
|
||||||
|
years.append(match.group(1))
|
||||||
|
return list(dict.fromkeys(years))
|
||||||
|
|
||||||
|
|
||||||
|
def answer_has_out_of_window_dates_for_scope(text: Any, date_scope: Any) -> bool:
|
||||||
|
expected_year = expected_single_year_from_date_scope(date_scope)
|
||||||
|
if not expected_year:
|
||||||
|
return False
|
||||||
|
years = dated_evidence_years_from_answer(text)
|
||||||
|
return any(year != expected_year for year in years)
|
||||||
|
|
||||||
|
|
||||||
def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||||
scenario_tree = raw_contract.get("scenario_tree")
|
scenario_tree = raw_contract.get("scenario_tree")
|
||||||
if not isinstance(scenario_tree, dict):
|
if not isinstance(scenario_tree, dict):
|
||||||
|
|
@ -2867,7 +2902,12 @@ def is_validated_clean_meta_chat_answer(
|
||||||
return False
|
return False
|
||||||
semantic_tags = set(normalize_string_list(state.get("semantic_tags")))
|
semantic_tags = set(normalize_string_list(state.get("semantic_tags")))
|
||||||
allowed_tags = {
|
allowed_tags = {
|
||||||
|
"human_answer",
|
||||||
"meta_smalltalk",
|
"meta_smalltalk",
|
||||||
|
"mcp_discovery_gate_sanity",
|
||||||
|
"off_domain_living_chat",
|
||||||
|
"stale_replay_forbidden",
|
||||||
|
"context_boundary",
|
||||||
"company_selected",
|
"company_selected",
|
||||||
"organization_authority",
|
"organization_authority",
|
||||||
"meta_capability",
|
"meta_capability",
|
||||||
|
|
@ -3112,6 +3152,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
|
||||||
if current_date_scope and current_date_scope != required_filters["as_of_date"]:
|
if current_date_scope and current_date_scope != required_filters["as_of_date"]:
|
||||||
violated_invariants.append("wrong_date_scope_state")
|
violated_invariants.append("wrong_date_scope_state")
|
||||||
|
|
||||||
|
if answer_has_out_of_window_dates_for_scope(assistant_text, date_scope):
|
||||||
|
violated_invariants.append("out_of_window_date_in_answer")
|
||||||
|
warnings.append("out_of_window_date_in_answer")
|
||||||
|
|
||||||
if should_require_direct_answer(state):
|
if should_require_direct_answer(state):
|
||||||
if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer):
|
if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer):
|
||||||
violated_invariants.append("direct_answer_missing")
|
violated_invariants.append("direct_answer_missing")
|
||||||
|
|
@ -3193,8 +3237,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
|
||||||
state["clarification_answer_validated"] = clarification_validated
|
state["clarification_answer_validated"] = clarification_validated
|
||||||
state["missing_axis_clarification_validated"] = missing_axis_clarification_validated
|
state["missing_axis_clarification_validated"] = missing_axis_clarification_validated
|
||||||
state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated
|
state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated
|
||||||
|
effective_execution_status = "exact" if clean_meta_chat_validated else execution_status
|
||||||
|
state["execution_status"] = effective_execution_status
|
||||||
state["acceptance_status"] = acceptance_status_from_execution(
|
state["acceptance_status"] = acceptance_status_from_execution(
|
||||||
execution_status,
|
effective_execution_status,
|
||||||
hard_fail,
|
hard_fail,
|
||||||
(
|
(
|
||||||
bounded_validated
|
bounded_validated
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
|
@ -287,6 +288,182 @@ class AgentDetectorRunnerTests(unittest.TestCase):
|
||||||
self.assertEqual(len(evidence_paths), 1)
|
self.assertEqual(len(evidence_paths), 1)
|
||||||
self.assertIn("step_01_margin_root", evidence_paths[0])
|
self.assertIn("step_01_margin_root", evidence_paths[0])
|
||||||
|
|
||||||
|
def test_counterparty_value_flow_required_surface_scopes_to_current_net_step(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
artifact_dir = root / "run"
|
||||||
|
write_text(
|
||||||
|
artifact_dir
|
||||||
|
/ "scenarios"
|
||||||
|
/ "mixed_planner_counterparty_evidence_and_living_guard"
|
||||||
|
/ "steps"
|
||||||
|
/ "step_03_incoming_by_resolved_entity"
|
||||||
|
/ "output.md",
|
||||||
|
"Входящие денежные поступления по контрагенту Группа СВК за 2020: 12 093 465 руб.",
|
||||||
|
)
|
||||||
|
write_text(
|
||||||
|
artifact_dir
|
||||||
|
/ "scenarios"
|
||||||
|
/ "mixed_planner_counterparty_evidence_and_living_guard"
|
||||||
|
/ "steps"
|
||||||
|
/ "step_05_net_after_payout"
|
||||||
|
/ "output.md",
|
||||||
|
"По контрагенту Группа СВК за период 2020 получили 12 093 465 руб., заплатили 0 руб.; расчетное нетто в нашу сторону: 12 093 465 руб.",
|
||||||
|
)
|
||||||
|
registry_path = root / "detector_registry.json"
|
||||||
|
issue_catalog_path = root / "issue_catalog.json"
|
||||||
|
write_json(
|
||||||
|
registry_path,
|
||||||
|
{
|
||||||
|
"schema_version": "agent_detector_registry_v1",
|
||||||
|
"detectors": {
|
||||||
|
"counterparty_value_flow_required_surface": {
|
||||||
|
"kind": "answer_text_required_any",
|
||||||
|
"automation_level": "automatic",
|
||||||
|
"description": "Net value-flow answer must surface all money directions.",
|
||||||
|
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||||
|
"inputs": ["output.md"],
|
||||||
|
"check": {
|
||||||
|
"artifact_path_include_patterns": ["(?i)step_05_net_after_payout"],
|
||||||
|
"required_patterns_any": [
|
||||||
|
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
|
||||||
|
|
||||||
|
results = runner.build_detector_results(
|
||||||
|
artifact_dir,
|
||||||
|
detector_names=["counterparty_value_flow_required_surface"],
|
||||||
|
registry_path=registry_path,
|
||||||
|
issue_catalog_path=issue_catalog_path,
|
||||||
|
include_default_global=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(results["summary"]["status"], "pass")
|
||||||
|
self.assertEqual(results["results"][0]["status"], "pass")
|
||||||
|
evidence_paths = [item["path"] for item in results["results"][0]["evidence"]]
|
||||||
|
self.assertEqual(len(evidence_paths), 1)
|
||||||
|
self.assertIn("step_05_net_after_payout", evidence_paths[0])
|
||||||
|
|
||||||
|
def test_answer_text_shape_uses_business_first_step_review(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
artifact_dir = root / "run"
|
||||||
|
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
|
||||||
|
write_text(step_dir / "output.md", "Direct business answer first.")
|
||||||
|
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": True}})
|
||||||
|
registry_path = root / "detector_registry.json"
|
||||||
|
issue_catalog_path = root / "issue_catalog.json"
|
||||||
|
write_json(
|
||||||
|
registry_path,
|
||||||
|
{
|
||||||
|
"schema_version": "agent_detector_registry_v1",
|
||||||
|
"detectors": {
|
||||||
|
"first_line_not_direct_answer": {
|
||||||
|
"kind": "answer_text_shape",
|
||||||
|
"automation_level": "semi_automatic",
|
||||||
|
"description": "First line should be direct.",
|
||||||
|
"issue_codes": ["business_direct_answer_missing"],
|
||||||
|
"inputs": ["output.md"],
|
||||||
|
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
|
||||||
|
|
||||||
|
results = runner.build_detector_results(
|
||||||
|
artifact_dir,
|
||||||
|
detector_names=["first_line_not_direct_answer"],
|
||||||
|
registry_path=registry_path,
|
||||||
|
issue_catalog_path=issue_catalog_path,
|
||||||
|
include_default_global=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(results["summary"]["status"], "pass")
|
||||||
|
self.assertEqual(results["results"][0]["status"], "pass")
|
||||||
|
|
||||||
|
def test_answer_text_shape_fails_when_business_first_review_rejects_first_line(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
artifact_dir = root / "run"
|
||||||
|
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
|
||||||
|
write_text(step_dir / "output.md", "Let me inspect the route first.")
|
||||||
|
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": False}})
|
||||||
|
registry_path = root / "detector_registry.json"
|
||||||
|
issue_catalog_path = root / "issue_catalog.json"
|
||||||
|
write_json(
|
||||||
|
registry_path,
|
||||||
|
{
|
||||||
|
"schema_version": "agent_detector_registry_v1",
|
||||||
|
"detectors": {
|
||||||
|
"first_line_not_direct_answer": {
|
||||||
|
"kind": "answer_text_shape",
|
||||||
|
"automation_level": "semi_automatic",
|
||||||
|
"description": "First line should be direct.",
|
||||||
|
"issue_codes": ["business_direct_answer_missing"],
|
||||||
|
"inputs": ["output.md"],
|
||||||
|
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
|
||||||
|
|
||||||
|
results = runner.build_detector_results(
|
||||||
|
artifact_dir,
|
||||||
|
detector_names=["first_line_not_direct_answer"],
|
||||||
|
registry_path=registry_path,
|
||||||
|
issue_catalog_path=issue_catalog_path,
|
||||||
|
include_default_global=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(results["summary"]["status"], "fail")
|
||||||
|
self.assertEqual(results["results"][0]["status"], "fail")
|
||||||
|
|
||||||
|
def test_step_state_lookup_handles_long_artifact_paths(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
step_dir = (
|
||||||
|
root
|
||||||
|
/ "run"
|
||||||
|
/ "scenarios"
|
||||||
|
/ ("mixed_planner_counterparty_evidence_and_living_guard_" + "s" * 48)
|
||||||
|
/ "steps"
|
||||||
|
/ ("step_04_payout_switch_by_resolved_entity_" + "x" * 80)
|
||||||
|
)
|
||||||
|
output_path = step_dir / "output.md"
|
||||||
|
step_state_path = step_dir / "step_state.json"
|
||||||
|
try:
|
||||||
|
os.makedirs(runner.path_for_io(step_dir), exist_ok=True)
|
||||||
|
with open(runner.path_for_io(output_path), "w", encoding="utf-8") as handle:
|
||||||
|
handle.write("Direct business answer first.")
|
||||||
|
runner.write_json(step_state_path, {"business_first_review": {"direct_answer_first_ok": True}})
|
||||||
|
|
||||||
|
self.assertGreater(len(str(step_state_path.absolute())), 260)
|
||||||
|
self.assertEqual(runner.output_step_state_path(output_path), step_state_path)
|
||||||
|
self.assertEqual(
|
||||||
|
runner.read_json_object(step_state_path).get("business_first_review"),
|
||||||
|
{"direct_answer_first_ok": True},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
for path in (output_path, step_state_path):
|
||||||
|
try:
|
||||||
|
os.remove(runner.path_for_io(path))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
current = step_dir
|
||||||
|
while current != root:
|
||||||
|
try:
|
||||||
|
os.rmdir(runner.path_for_io(current))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
current = current.parent
|
||||||
|
|
||||||
def test_composite_detector_fails_after_child_detector_fails(self) -> None:
|
def test_composite_detector_fails_after_child_detector_fails(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
root = Path(tmp)
|
root = Path(tmp)
|
||||||
|
|
|
||||||
|
|
@ -153,6 +153,81 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
|
||||||
self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial")
|
self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial")
|
||||||
self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted")
|
self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted")
|
||||||
|
|
||||||
|
def test_clean_meta_smalltalk_is_exact_without_1c_capability(self) -> None:
|
||||||
|
step_state = dcl.build_scenario_step_state(
|
||||||
|
scenario_id="meta_chat_demo",
|
||||||
|
domain="agentic_loop",
|
||||||
|
step={
|
||||||
|
"step_id": "step_01",
|
||||||
|
"title": "Human smalltalk",
|
||||||
|
"depends_on": [],
|
||||||
|
"question_template": "привет, ты на связи?",
|
||||||
|
"semantic_tags": ["human_answer", "meta_smalltalk", "mcp_discovery_gate_sanity"],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
},
|
||||||
|
step_index=1,
|
||||||
|
question_resolved="привет, ты на связи?",
|
||||||
|
analysis_context={},
|
||||||
|
turn_artifact={
|
||||||
|
"assistant_message": {
|
||||||
|
"reply_type": "factual_with_explanation",
|
||||||
|
"text": "Привет! Да, я на связи. Готов помочь с анализом данных из 1С в режиме чтения.",
|
||||||
|
"message_id": "msg-1",
|
||||||
|
"trace_id": "trace-1",
|
||||||
|
},
|
||||||
|
"technical_debug_payload": {
|
||||||
|
"detected_mode": "chat",
|
||||||
|
"fallback_type": "none",
|
||||||
|
"living_chat_response_source": "llm_chat",
|
||||||
|
},
|
||||||
|
"session_summary": {},
|
||||||
|
},
|
||||||
|
entries=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
|
||||||
|
self.assertEqual(step_state["execution_status"], "exact")
|
||||||
|
self.assertEqual(step_state["acceptance_status"], "validated")
|
||||||
|
|
||||||
|
def test_off_domain_living_chat_is_exact_without_1c_capability(self) -> None:
|
||||||
|
step_state = dcl.build_scenario_step_state(
|
||||||
|
scenario_id="off_domain_demo",
|
||||||
|
domain="agentic_loop",
|
||||||
|
step={
|
||||||
|
"step_id": "step_09",
|
||||||
|
"title": "Off-domain living chat",
|
||||||
|
"depends_on": [],
|
||||||
|
"question_template": "а чем капибара отличается от утки?",
|
||||||
|
"semantic_tags": ["off_domain_living_chat", "stale_replay_forbidden", "context_boundary"],
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
},
|
||||||
|
step_index=9,
|
||||||
|
question_resolved="а чем капибара отличается от утки?",
|
||||||
|
analysis_context={},
|
||||||
|
turn_artifact={
|
||||||
|
"assistant_message": {
|
||||||
|
"reply_type": "factual_with_explanation",
|
||||||
|
"text": (
|
||||||
|
"Капибара и утка отличаются принципиально: капибара - млекопитающее-грызун, "
|
||||||
|
"а утка - птица. Поэтому у них разные тело, среда обитания и способ передвижения."
|
||||||
|
),
|
||||||
|
"message_id": "msg-2",
|
||||||
|
"trace_id": "trace-2",
|
||||||
|
},
|
||||||
|
"technical_debug_payload": {
|
||||||
|
"detected_mode": "chat",
|
||||||
|
"fallback_type": "none",
|
||||||
|
"living_chat_response_source": "llm_chat",
|
||||||
|
},
|
||||||
|
"session_summary": {},
|
||||||
|
},
|
||||||
|
entries=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
|
||||||
|
self.assertEqual(step_state["execution_status"], "exact")
|
||||||
|
self.assertEqual(step_state["acceptance_status"], "validated")
|
||||||
|
|
||||||
def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None:
|
def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None:
|
||||||
self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?"))
|
self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?"))
|
||||||
|
|
||||||
|
|
@ -610,6 +685,32 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
|
||||||
self.assertIsNone(step_state["date_scope"]["as_of_date"])
|
self.assertIsNone(step_state["date_scope"]["as_of_date"])
|
||||||
self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset")
|
self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset")
|
||||||
|
|
||||||
|
def test_out_of_window_document_dates_reject_validated_step(self) -> None:
|
||||||
|
validated = dcl.validate_step_contract(
|
||||||
|
{
|
||||||
|
"execution_status": "exact",
|
||||||
|
"required_answer_shape": "direct_answer_first",
|
||||||
|
"reply_type": "factual",
|
||||||
|
"actual_direct_answer": "Контрагент: Группа СВК. Найдено документов: 19.",
|
||||||
|
"assistant_text": (
|
||||||
|
"Контрагент: Группа СВК. Найдено документов: 19.\n"
|
||||||
|
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07"
|
||||||
|
),
|
||||||
|
"top_non_empty_lines": [
|
||||||
|
"Контрагент: Группа СВК. Найдено документов: 19.",
|
||||||
|
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07",
|
||||||
|
],
|
||||||
|
"date_scope": {
|
||||||
|
"period_from": "2020-01-01",
|
||||||
|
"period_to": "2020-12-31",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(validated["status"], "rejected")
|
||||||
|
self.assertIn("out_of_window_date_in_answer", validated["violated_invariants"])
|
||||||
|
self.assertIn("out_of_window_date_in_answer", validated["warnings"])
|
||||||
|
|
||||||
def test_open_items_exact_negative_answer_validates_without_rows(self) -> None:
|
def test_open_items_exact_negative_answer_validates_without_rows(self) -> None:
|
||||||
step_state = dcl.build_scenario_step_state(
|
step_state = dcl.build_scenario_step_state(
|
||||||
scenario_id="open_items_negative_demo",
|
scenario_id="open_items_negative_demo",
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue