Укрепить агентный loop Phase107

This commit is contained in:
dctouch 2026-06-15 22:56:22 +03:00
parent b3f2e405bb
commit 41463c9965
6 changed files with 718 additions and 140 deletions

View File

@ -1,10 +1,10 @@
{ {
"schema_version": "domain_scenario_pack_v1", "schema_version": "domain_scenario_pack_v1",
"pack_id": "active_phase106_profit_cashflow_margin_next_step_20260615", "pack_id": "active_phase107_mixed_planner_brain_dogfood_20260615",
"domain": "address_phase106_profit_cashflow_margin_next_step", "domain": "address_phase107_mixed_planner_brain_dogfood",
"runtime_domain": "business_overview_profit_cashflow_margin_boundary", "runtime_domain": "planner_autonomy_mixed_boundary",
"title": "Profit, cashflow, margin and next-check boundary", "title": "Mixed planner-brain dogfood replay",
"description": "Active slice for the autonomous agent loop: colloquial earnings wording must be answered as checked cashflow/accounting evidence with honest profit boundaries, then pivot safely into товарная маржа and concrete next checks.", "description": "Active slice for dogfooding the autonomous agent loop on a compact mixed planner path: living chat guard, counterparty alias grounding, follow-up money flow, net, documents, movement evidence, broad business evaluation, and off-domain recovery.",
"status": "active", "status": "active",
"source_of_truth_policy": { "source_of_truth_policy": {
"purpose": "single mutable domain source for the current orchestration target", "purpose": "single mutable domain source for the current orchestration target",
@ -21,182 +21,349 @@
}, },
"analysis_context": { "analysis_context": {
"as_of_date": "2026-06-15", "as_of_date": "2026-06-15",
"source": "phase106_after_phase104_role_tail_autorun_persistence", "source": "phase107_after_phase106_profit_cashflow_margin_next_step",
"known_previous_anchors": [ "known_previous_anchors": [
"hm_business_overview_cashflow_limits_20260602_p4", "phase83_planner_brain_alignment_mix_20260603_live9",
"hm_business_profile_evaluation_limits_p3", "phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop",
"agent_margin_profitability_reliability_20260615_p12_detector_scope", "phase106_profit_cashflow_margin_next_step_20260615_p03_loop"
"phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop" ],
] "why_now": "Phase83 passed in the legacy truth-harness format; this pack replays the highest-value mixed planner slice through the current autonomous pack-loop."
}, },
"target_score": 88, "target_score": 88,
"bindings": { "bindings": {
"main_organization": "ООО Альтернатива Плюс", "main_organization": "ООО Альтернатива Плюс",
"control_year": "2020" "control_year": "2020",
"svk_counterparty": "Группа СВК"
}, },
"issue_codes_under_test": [ "issue_codes_under_test": [
"cashflow_misreported_as_clean_profit", "technical_garbage_in_answer",
"margin_domain_leak_accounting_route", "business_direct_answer_missing",
"business_next_step_missing", "business_next_step_missing",
"technical_garbage_in_answer" "counterparty_value_flow_misrouted_to_company_profit",
"business_utility_gap"
], ],
"detectors_under_test": [ "detectors_under_test": [
"runtime_tokens_in_user_answer", "runtime_tokens_in_user_answer",
"capability_ids_in_user_answer", "capability_ids_in_user_answer",
"first_line_not_direct_answer",
"top_level_scaffold_before_answer",
"limited_answer_without_next_action", "limited_answer_without_next_action",
"margin_domain_leak_accounting_route", "counterparty_value_flow_required_surface",
"margin_required_fields_missing", "counterparty_value_flow_profit_accounts_forbidden"
"margin_next_action_missing"
], ],
"agent_audit_expectations": { "agent_audit_expectations": {
"semantic_answer_review_first": true, "semantic_answer_review_first": true,
"direct_answer_first": true, "direct_answer_first": true,
"business_utility_required": true, "business_utility_required": true,
"cashflow_not_clean_profit": true, "selected_counterparty_memory_required": true,
"margin_domain_purity_required": true, "date_carryover_required": true,
"next_action_required_when_limited": true, "documents_and_movements_are_evidence_drilldowns": true,
"off_domain_living_chat_must_not_replay_business_context": true,
"technical_garbage_forbidden": true, "technical_garbage_forbidden": true,
"do_not_accept_if": [ "do_not_accept_if": [
"colloquial earnings wording is answered as confirmed clean profit without evidence", "a follow-up over 'нему' loses the selected counterparty",
"cashflow/net operating flow is silently presented as accounting profit or margin", "a payout or net follow-up loses the 2020 period",
"margin follow-up leaks into bank payments, fixed assets, amortization, or generic accounting route", "a counterparty received/paid/net question is answered as company profit",
"next-check answer is generic advice without concrete 1C evidence contours", "document or movement follow-up answers with a fresh clarification instead of reusing the selected counterparty",
"final answer leaks route ids, capability ids, runtime enums, or debug payload" "broad business evaluation leaks route ids, capability ids, runtime enums, or debug payload",
"off-domain living chat is hijacked by stale 1C business context"
] ]
}, },
"scenarios": [ "scenarios": [
{ {
"scenario_id": "profit_cashflow_margin_next_step_boundary", "scenario_id": "mixed_planner_counterparty_evidence_and_living_guard",
"title": "Colloquial earnings, clean-profit boundary, margin pivot, and next checks", "title": "Counterparty memory, evidence drilldowns, business overview, and off-domain guard",
"description": "Validate that the assistant handles colloquial earnings wording, refuses to overclaim clean profit, pivots into товарная маржа without wrong-domain leakage, and proposes concrete next 1C checks.", "description": "Validate a compact but cross-cutting planner path: chat sanity, loose counterparty alias grounding, follow-up money flow and net, documents/movements evidence, explicit broad company evaluation, and recovery into ordinary living chat.",
"steps": [ "steps": [
{ {
"step_id": "step_01_choose_company_scope", "step_id": "step_01_human_smalltalk_sanity",
"title": "Bare organization choice binds active company scope", "title": "Living chat remains human and does not expose discovery internals",
"node_role": "root", "node_role": "root",
"question": "{{bindings.main_organization}}", "question": "привет, ты на связи?",
"allowed_reply_types": [ "allowed_reply_types": [
"factual_with_explanation", "factual_with_explanation",
"partial_coverage" "partial_coverage"
], ],
"semantic_tags": [ "semantic_tags": [
"bare_org_scope", "human_answer",
"company_scope" "mcp_discovery_gate_sanity",
], "meta_smalltalk"
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)фиксир|рабоч.*организац|контур",
"(?i)альтернатива"
],
"forbidden_answer_patterns": [
"(?i)не могу определить",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object"
]
},
{
"step_id": "step_02_colloquial_earned_money_2020",
"title": "Colloquial earned-money wording stays bounded",
"question": "скока денег альтернатива заработала за 20 год?",
"depends_on": [
"step_01_choose_company_scope"
],
"semantic_tags": [
"business_overview",
"colloquial_cashflow",
"cashflow_not_profit"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020|20\\s*год",
"(?i)поступ|входящ|денег|заработ",
"(?i)не.*чист.*прибыл|не.*прибыл|прибыл.*не подтвержд|это.*не.*прибыл"
],
"forbidden_answer_patterns": [
"(?i)^\\s*Коротко:",
"(?i)это\\s+чистая\\s+прибыль",
"(?i)точно.*прибыл",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_03_clean_profit_followup_boundary",
"title": "Clean-profit follow-up must not convert cashflow into profit",
"question": "а это чистая прибыль?",
"depends_on": [
"step_02_colloquial_earned_money_2020"
],
"semantic_tags": [
"profitability_boundary",
"cashflow_not_profit",
"followup_boundary"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)не.*чист.*прибыл|не.*является.*прибыл|не подтвержд.*прибыл|это.*не.*прибыл",
"(?i)денежн|поступ|нетто|оборот|cash|себестоим|расход|закрыва"
],
"forbidden_answer_patterns": [
"(?i)^\\s*Коротко:",
"(?i)^\\s*да[,\\s]+это.*прибыл",
"(?i)точно.*чист.*прибыл",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_04_margin_pivot_keeps_inventory_domain",
"title": "Margin pivot uses inventory margin contour, not payments or fixed assets",
"question": "тогда какую прибыль или маржу можно проверить по товарам за 2020?",
"depends_on": [
"step_03_clean_profit_followup_boundary"
],
"expected_intents": [
"inventory_margin_ranking_for_nomenclature"
],
"expected_capability": "inventory_inventory_margin_ranking_for_nomenclature",
"expected_recipe": "address_inventory_margin_ranking_for_nomenclature_v1",
"semantic_tags": [
"margin_profitability",
"inventory",
"wrong_domain_trap",
"cashflow_to_margin_pivot"
], ],
"required_answer_shape": "direct_answer_first", "required_answer_shape": "direct_answer_first",
"required_answer_patterns_any": [ "required_answer_patterns_any": [
"(?i)марж|валов|выруч|себестоим|номенклатур|товар|не могу подтвердить|не хватает" "(?i)привет|на связи|готов|помочь"
], ],
"forbidden_answer_patterns": [ "forbidden_answer_patterns": [
"(?i)амортизац|основн(ые|ых)? средств|объект ОС|банк|оплат[аы]|payment_document|settlement", "(?i)mcp|runtime_|query_documents|primitive|route_candidate|capability_id|snapshot_items|answer_object"
],
"criticality": "info"
},
{
"step_id": "step_02_resolve_counterparty_alias",
"title": "Entity resolution grounds the checked 1C counterparty from a loose alias",
"question": "найди в 1С контрагента СВК",
"depends_on": [
"step_01_human_smalltalk_sanity"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"alias_grounding",
"followup_anchor",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)свк",
"(?i)контрагент"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)каталог|найден|наиболее вероятн"
],
"forbidden_answer_patterns": [
"(?i)получили|заплатили|нетто|оборот|выручк|сумм(а|ы)",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
], ],
"criticality": "critical" "criticality": "critical"
}, },
{ {
"step_id": "step_05_next_checks_are_concrete_1c_contours", "step_id": "step_03_incoming_by_resolved_entity",
"title": "Next checks must be concrete 1C evidence contours", "title": "Incoming value-flow follow-up reuses the resolved counterparty anchor",
"question": "что дальше проверить, чтобы понять здоровье бизнеса, без гаданий?", "question": "сколько получили по нему за 2020 год",
"depends_on": [ "depends_on": [
"step_04_margin_pivot_keeps_inventory_domain" "step_02_resolve_counterparty_alias"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
], ],
"semantic_tags": [ "semantic_tags": [
"business_next_step", "entity_resolution",
"evidence_plan", "incoming_value_flow",
"no_generic_advice" "followup_reuse",
"date_carryover",
"planner_catalog_alignment"
], ],
"required_answer_shape": "direct_answer_first", "required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [ "required_answer_patterns_all": [
"(?i)прибыл|марж|себестоим|финрезульт", "(?i)2020",
"(?i)ндс|налог", "(?i)получил|входящ|поступ",
"(?i)дебитор|кредитор|долг|расчет", "(?i)руб"
"(?i)склад|остат|оборач|товар" ],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк"
], ],
"forbidden_answer_patterns": [ "forbidden_answer_patterns": [
"(?i)^\\s*Коротко:", "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)просто.*посмотр|общ.*рекоменд|улучшить маркетинг|нанять|стратег", "(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
], ],
"criticality": "important" "criticality": "critical"
},
{
"step_id": "step_04_payout_switch_by_resolved_entity",
"title": "Outgoing payment follow-up keeps the same grounded counterparty and checked year",
"question": "а теперь сколько заплатили?",
"depends_on": [
"step_03_incoming_by_resolved_entity"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"payout_switch",
"followup_reuse",
"date_carryover",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020",
"(?i)заплатил|исходящ|списан|платеж|платёж",
"(?i)руб"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
"(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_05_net_after_payout",
"title": "Net-flow follow-up reuses the same grounded counterparty and checked year after payout",
"question": "а какое нетто?",
"depends_on": [
"step_04_payout_switch_by_resolved_entity"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"net_value_flow",
"followup_reuse",
"date_carryover",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020",
"(?i)нетто|сальдо|разниц",
"(?i)получ",
"(?i)заплат",
"(?i)руб"
],
"required_answer_patterns_any": [
"12[ .\\u00a0]?093[ .\\u00a0]?465|12093465",
"(?i)группа\\s+свк|свк"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
"(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_06_documents_after_net",
"title": "Document evidence follow-up keeps the grounded counterparty after the net answer",
"question": "а по документам?",
"depends_on": [
"step_05_net_after_payout"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"document_evidence",
"value_flow_pivot",
"followup_reuse",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)документ|счет|счёт|накладн|акт"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк",
"(?i)2020"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)сколько получили|сколько заплатили|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_07_movements_after_documents",
"title": "Movement evidence follow-up keeps the grounded counterparty after the document answer",
"question": "а по движениям?",
"depends_on": [
"step_06_documents_after_net"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"movement_evidence",
"document_pivot",
"followup_reuse",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)движени|операц|платеж|платёж|списан|поступ"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк",
"(?i)2020"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)сколько получили|сколько заплатили|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_08_broad_business_evaluation_explicit_org",
"title": "Broad business evaluation stays grounded and bounded after counterparty drilldowns",
"question": "Как ты оценишь деятельность компании {{bindings.main_organization}} по данным 1С?",
"depends_on": [
"step_07_movements_after_documents"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"broad_business_evaluation",
"grounded_summary",
"context_switch_from_counterparty_to_company"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)альтернатива",
"(?i)ограниченн|проверенн|не\\s+аудит|не\\s+аудитор",
"(?i)1с|подтвержд",
"(?i)денежн|долг|ндс|контрагент|операц"
],
"forbidden_answer_patterns": [
"(?i)^\\s*коротко:",
"(?i)чистая\\s+прибыль\\s+подтверждена|аудиторское\\s+заключение",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_09_off_domain_living_chat_not_hijacked",
"title": "Off-domain living chat is not hijacked by stale business context",
"question": "а чем капибара отличается от утки?",
"depends_on": [
"step_08_broad_business_evaluation_explicit_org"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"off_domain_living_chat",
"stale_replay_forbidden",
"context_boundary"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_any": [
"(?i)капибар.*утк|утк.*капибар",
"(?i)млекопита|птиц|грызун"
],
"forbidden_answer_patterns": [
"(?i)альтернатива|свк|контрагент|1с|поступлен|платеж|платёж|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "warning"
} }
] ]
} }

View File

@ -218,7 +218,9 @@
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"], "issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["steps/<step_id>/output.md"], "inputs": ["steps/<step_id>/output.md"],
"check": { "check": {
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"], "artifact_path_include_patterns": [
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_05_net_after_payout)"
],
"required_patterns_any": [ "required_patterns_any": [
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))" "(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
] ]
@ -231,7 +233,9 @@
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"], "issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["steps/<step_id>/output.md"], "inputs": ["steps/<step_id>/output.md"],
"check": { "check": {
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"], "artifact_path_include_patterns": [
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_03_incoming_by_resolved_entity|step_04_payout_switch_by_resolved_entity|step_05_net_after_payout)"
],
"forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"] "forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"]
} }
}, },

View File

@ -2,6 +2,7 @@ from __future__ import annotations
import argparse import argparse
import json import json
import os
import re import re
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
@ -18,8 +19,35 @@ DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS = [
] ]
def path_for_io(path: Path) -> str:
if os.name != "nt":
return str(path)
raw_value = str(path)
if raw_value.startswith("\\\\?\\"):
return raw_value
absolute_path = path if path.is_absolute() else Path.cwd() / path
absolute_value = str(absolute_path.absolute())
if absolute_value.startswith("\\\\?\\"):
return absolute_value
if absolute_value.startswith("\\\\"):
return "\\\\?\\UNC\\" + absolute_value.lstrip("\\")
return "\\\\?\\" + absolute_value
def path_exists(path: Path) -> bool:
try:
return os.path.exists(path_for_io(path))
except OSError:
return False
def read_text(path: Path) -> str:
with open(path_for_io(path), "r", encoding="utf-8") as handle:
return handle.read()
def read_json(path: Path) -> Any: def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8")) return json.loads(read_text(path))
def read_json_object(path: Path) -> dict[str, Any]: def read_json_object(path: Path) -> dict[str, Any]:
@ -31,8 +59,9 @@ def read_json_object(path: Path) -> dict[str, Any]:
def write_json(path: Path, payload: Any) -> None: def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True) os.makedirs(path_for_io(path.parent), exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") with open(path_for_io(path), "w", encoding="utf-8") as handle:
handle.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n")
def utc_now() -> str: def utc_now() -> str:
@ -164,7 +193,7 @@ def select_detectors(
def read_text_or_empty(path: Path) -> str: def read_text_or_empty(path: Path) -> str:
try: try:
return path.read_text(encoding="utf-8") return read_text(path)
except OSError: except OSError:
return "" return ""
@ -207,7 +236,7 @@ def first_pattern_search(patterns: list[re.Pattern[str]], text: str) -> re.Match
def assistant_text_from_turn_path(path: Path | None) -> str: def assistant_text_from_turn_path(path: Path | None) -> str:
if path is None or not path.exists(): if path is None or not path_exists(path):
return "" return ""
payload = read_json_object(path) payload = read_json_object(path)
assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {} assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {}
@ -264,14 +293,21 @@ def output_turn_path(output_path: Path) -> Path | None:
name = output_path.name name = output_path.name
if name == "output.md": if name == "output.md":
candidate = output_path.with_name("turn.json") candidate = output_path.with_name("turn.json")
return candidate if candidate.exists() else None return candidate if path_exists(candidate) else None
if name.endswith("_output.md"): if name.endswith("_output.md"):
prefix = name[: -len("_output.md")] prefix = name[: -len("_output.md")]
candidate = output_path.with_name(f"{prefix}_turn.json") candidate = output_path.with_name(f"{prefix}_turn.json")
return candidate if candidate.exists() else None return candidate if path_exists(candidate) else None
return None return None
def output_step_state_path(output_path: Path) -> Path | None:
if output_path.name != "output.md":
return None
candidate = output_path.with_name("step_state.json")
return candidate if path_exists(candidate) else None
def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]: def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
outputs: list[dict[str, Any]] = [] outputs: list[dict[str, Any]] = []
seen: set[Path] = set() seen: set[Path] = set()
@ -292,6 +328,7 @@ def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
"artifact_path": str(path.relative_to(artifact_dir)), "artifact_path": str(path.relative_to(artifact_dir)),
"text": read_text_or_empty(path), "text": read_text_or_empty(path),
"turn_path": turn_path, "turn_path": turn_path,
"step_state_path": output_step_state_path(path),
} }
) )
return outputs return outputs
@ -497,6 +534,52 @@ def evaluate_limited_next_action(
return build_result(detector_name, detector, status, message, evidence=failures) return build_result(detector_name, detector, status, message, evidence=failures)
def evaluate_answer_text_shape(
detector_name: str,
detector: dict[str, Any],
outputs: list[dict[str, Any]],
) -> dict[str, Any]:
if not outputs:
return build_result(detector_name, detector, "skipped", "no output.md-style artifacts matched detector scope")
failures: list[dict[str, Any]] = []
reviewed: list[dict[str, Any]] = []
unknown: list[dict[str, Any]] = []
for output in outputs:
step_state_path = output.get("step_state_path")
step_state = read_json_object(step_state_path) if isinstance(step_state_path, Path) else {}
review = step_state.get("business_first_review") if isinstance(step_state.get("business_first_review"), dict) else {}
direct_answer_first_ok = review.get("direct_answer_first_ok")
if direct_answer_first_ok is True:
reviewed.append({"path": output["repo_path"], "direct_answer_first_ok": True})
elif direct_answer_first_ok is False:
failures.append({"path": output["repo_path"], "direct_answer_first_ok": False})
else:
unknown.append({"path": output["repo_path"], "reason": "business_first_review_missing"})
if failures:
return build_result(
detector_name,
detector,
"fail",
"first-line direct answer check failed",
evidence=[*failures, *unknown],
)
if unknown:
return build_result(
detector_name,
detector,
"review",
"direct-answer shape requires business review",
evidence=[*reviewed, *unknown],
)
return build_result(
detector_name,
detector,
"pass",
"business-first step reviews confirm direct-answer-first shape",
evidence=reviewed,
)
def evaluate_trace_guard( def evaluate_trace_guard(
detector_name: str, detector_name: str,
detector: dict[str, Any], detector: dict[str, Any],
@ -686,7 +769,7 @@ def evaluate_detector(
): ):
return evaluate_composite(detector_name, detector, results_by_name) return evaluate_composite(detector_name, detector, results_by_name)
if kind == "answer_text_shape": if kind == "answer_text_shape":
return evaluate_manual_review(detector_name, detector, "direct-answer shape requires business review") return evaluate_answer_text_shape(detector_name, detector, scoped_outputs)
return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}") return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}")

View File

@ -317,6 +317,7 @@ DEFAULT_INVARIANT_SEVERITY: dict[str, str] = {
"forbidden_recipe_selected": "P0", "forbidden_recipe_selected": "P0",
"focus_object_missing": "P0", "focus_object_missing": "P0",
"wrong_date_scope_state": "P0", "wrong_date_scope_state": "P0",
"out_of_window_date_in_answer": "P0",
"direct_answer_missing": "P0", "direct_answer_missing": "P0",
"top_level_noise_present": "P0", "top_level_noise_present": "P0",
"business_direct_answer_missing": "P0", "business_direct_answer_missing": "P0",
@ -930,6 +931,40 @@ def first_non_empty_lines(text: str, limit: int = 3) -> list[str]:
return output return output
def expected_single_year_from_date_scope(date_scope: Any) -> str | None:
if not isinstance(date_scope, dict):
return None
if str(date_scope.get("scope") or "").strip().lower() == "all_time":
return None
period_from = normalize_iso_date(date_scope.get("period_from"))
period_to = normalize_iso_date(date_scope.get("period_to"))
if not period_from or not period_to:
return None
from_year = period_from[:4]
to_year = period_to[:4]
if from_year != to_year:
return None
return from_year if re.fullmatch(r"(?:19|20)\d{2}", from_year) else None
def dated_evidence_years_from_answer(text: Any) -> list[str]:
source = repair_text_mojibake(str(text or ""))
years: list[str] = []
for match in re.finditer(r"\b((?:19|20)\d{2})-\d{2}-\d{2}(?:T|\b)", source):
years.append(match.group(1))
for match in re.finditer(r"\b\d{1,2}\.\d{1,2}\.((?:19|20)\d{2})\b", source):
years.append(match.group(1))
return list(dict.fromkeys(years))
def answer_has_out_of_window_dates_for_scope(text: Any, date_scope: Any) -> bool:
expected_year = expected_single_year_from_date_scope(date_scope)
if not expected_year:
return False
years = dated_evidence_years_from_answer(text)
return any(year != expected_year for year in years)
def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]: def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]:
scenario_tree = raw_contract.get("scenario_tree") scenario_tree = raw_contract.get("scenario_tree")
if not isinstance(scenario_tree, dict): if not isinstance(scenario_tree, dict):
@ -2867,7 +2902,12 @@ def is_validated_clean_meta_chat_answer(
return False return False
semantic_tags = set(normalize_string_list(state.get("semantic_tags"))) semantic_tags = set(normalize_string_list(state.get("semantic_tags")))
allowed_tags = { allowed_tags = {
"human_answer",
"meta_smalltalk", "meta_smalltalk",
"mcp_discovery_gate_sanity",
"off_domain_living_chat",
"stale_replay_forbidden",
"context_boundary",
"company_selected", "company_selected",
"organization_authority", "organization_authority",
"meta_capability", "meta_capability",
@ -3112,6 +3152,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
if current_date_scope and current_date_scope != required_filters["as_of_date"]: if current_date_scope and current_date_scope != required_filters["as_of_date"]:
violated_invariants.append("wrong_date_scope_state") violated_invariants.append("wrong_date_scope_state")
if answer_has_out_of_window_dates_for_scope(assistant_text, date_scope):
violated_invariants.append("out_of_window_date_in_answer")
warnings.append("out_of_window_date_in_answer")
if should_require_direct_answer(state): if should_require_direct_answer(state):
if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer): if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer):
violated_invariants.append("direct_answer_missing") violated_invariants.append("direct_answer_missing")
@ -3193,8 +3237,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
state["clarification_answer_validated"] = clarification_validated state["clarification_answer_validated"] = clarification_validated
state["missing_axis_clarification_validated"] = missing_axis_clarification_validated state["missing_axis_clarification_validated"] = missing_axis_clarification_validated
state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated
effective_execution_status = "exact" if clean_meta_chat_validated else execution_status
state["execution_status"] = effective_execution_status
state["acceptance_status"] = acceptance_status_from_execution( state["acceptance_status"] = acceptance_status_from_execution(
execution_status, effective_execution_status,
hard_fail, hard_fail,
( (
bounded_validated bounded_validated

View File

@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import json import json
import os
import sys import sys
import tempfile import tempfile
import unittest import unittest
@ -287,6 +288,182 @@ class AgentDetectorRunnerTests(unittest.TestCase):
self.assertEqual(len(evidence_paths), 1) self.assertEqual(len(evidence_paths), 1)
self.assertIn("step_01_margin_root", evidence_paths[0]) self.assertIn("step_01_margin_root", evidence_paths[0])
def test_counterparty_value_flow_required_surface_scopes_to_current_net_step(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
write_text(
artifact_dir
/ "scenarios"
/ "mixed_planner_counterparty_evidence_and_living_guard"
/ "steps"
/ "step_03_incoming_by_resolved_entity"
/ "output.md",
"Входящие денежные поступления по контрагенту Группа СВК за 2020: 12 093 465 руб.",
)
write_text(
artifact_dir
/ "scenarios"
/ "mixed_planner_counterparty_evidence_and_living_guard"
/ "steps"
/ "step_05_net_after_payout"
/ "output.md",
"По контрагенту Группа СВК за период 2020 получили 12 093 465 руб., заплатили 0 руб.; расчетное нетто в нашу сторону: 12 093 465 руб.",
)
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"counterparty_value_flow_required_surface": {
"kind": "answer_text_required_any",
"automation_level": "automatic",
"description": "Net value-flow answer must surface all money directions.",
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["output.md"],
"check": {
"artifact_path_include_patterns": ["(?i)step_05_net_after_payout"],
"required_patterns_any": [
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
],
},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["counterparty_value_flow_required_surface"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
evidence_paths = [item["path"] for item in results["results"][0]["evidence"]]
self.assertEqual(len(evidence_paths), 1)
self.assertIn("step_05_net_after_payout", evidence_paths[0])
def test_answer_text_shape_uses_business_first_step_review(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
write_text(step_dir / "output.md", "Direct business answer first.")
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": True}})
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"first_line_not_direct_answer": {
"kind": "answer_text_shape",
"automation_level": "semi_automatic",
"description": "First line should be direct.",
"issue_codes": ["business_direct_answer_missing"],
"inputs": ["output.md"],
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["first_line_not_direct_answer"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
def test_answer_text_shape_fails_when_business_first_review_rejects_first_line(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
write_text(step_dir / "output.md", "Let me inspect the route first.")
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": False}})
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"first_line_not_direct_answer": {
"kind": "answer_text_shape",
"automation_level": "semi_automatic",
"description": "First line should be direct.",
"issue_codes": ["business_direct_answer_missing"],
"inputs": ["output.md"],
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["first_line_not_direct_answer"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "fail")
self.assertEqual(results["results"][0]["status"], "fail")
def test_step_state_lookup_handles_long_artifact_paths(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
step_dir = (
root
/ "run"
/ "scenarios"
/ ("mixed_planner_counterparty_evidence_and_living_guard_" + "s" * 48)
/ "steps"
/ ("step_04_payout_switch_by_resolved_entity_" + "x" * 80)
)
output_path = step_dir / "output.md"
step_state_path = step_dir / "step_state.json"
try:
os.makedirs(runner.path_for_io(step_dir), exist_ok=True)
with open(runner.path_for_io(output_path), "w", encoding="utf-8") as handle:
handle.write("Direct business answer first.")
runner.write_json(step_state_path, {"business_first_review": {"direct_answer_first_ok": True}})
self.assertGreater(len(str(step_state_path.absolute())), 260)
self.assertEqual(runner.output_step_state_path(output_path), step_state_path)
self.assertEqual(
runner.read_json_object(step_state_path).get("business_first_review"),
{"direct_answer_first_ok": True},
)
finally:
for path in (output_path, step_state_path):
try:
os.remove(runner.path_for_io(path))
except OSError:
pass
current = step_dir
while current != root:
try:
os.rmdir(runner.path_for_io(current))
except OSError:
pass
current = current.parent
def test_composite_detector_fails_after_child_detector_fails(self) -> None: def test_composite_detector_fails_after_child_detector_fails(self) -> None:
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp) root = Path(tmp)

View File

@ -153,6 +153,81 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial") self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial")
self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted") self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted")
def test_clean_meta_smalltalk_is_exact_without_1c_capability(self) -> None:
step_state = dcl.build_scenario_step_state(
scenario_id="meta_chat_demo",
domain="agentic_loop",
step={
"step_id": "step_01",
"title": "Human smalltalk",
"depends_on": [],
"question_template": "привет, ты на связи?",
"semantic_tags": ["human_answer", "meta_smalltalk", "mcp_discovery_gate_sanity"],
"required_answer_shape": "direct_answer_first",
},
step_index=1,
question_resolved="привет, ты на связи?",
analysis_context={},
turn_artifact={
"assistant_message": {
"reply_type": "factual_with_explanation",
"text": "Привет! Да, я на связи. Готов помочь с анализом данных из 1С в режиме чтения.",
"message_id": "msg-1",
"trace_id": "trace-1",
},
"technical_debug_payload": {
"detected_mode": "chat",
"fallback_type": "none",
"living_chat_response_source": "llm_chat",
},
"session_summary": {},
},
entries=[],
)
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
self.assertEqual(step_state["execution_status"], "exact")
self.assertEqual(step_state["acceptance_status"], "validated")
def test_off_domain_living_chat_is_exact_without_1c_capability(self) -> None:
step_state = dcl.build_scenario_step_state(
scenario_id="off_domain_demo",
domain="agentic_loop",
step={
"step_id": "step_09",
"title": "Off-domain living chat",
"depends_on": [],
"question_template": "а чем капибара отличается от утки?",
"semantic_tags": ["off_domain_living_chat", "stale_replay_forbidden", "context_boundary"],
"required_answer_shape": "direct_answer_first",
},
step_index=9,
question_resolved="а чем капибара отличается от утки?",
analysis_context={},
turn_artifact={
"assistant_message": {
"reply_type": "factual_with_explanation",
"text": (
"Капибара и утка отличаются принципиально: капибара - млекопитающее-грызун, "
"а утка - птица. Поэтому у них разные тело, среда обитания и способ передвижения."
),
"message_id": "msg-2",
"trace_id": "trace-2",
},
"technical_debug_payload": {
"detected_mode": "chat",
"fallback_type": "none",
"living_chat_response_source": "llm_chat",
},
"session_summary": {},
},
entries=[],
)
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
self.assertEqual(step_state["execution_status"], "exact")
self.assertEqual(step_state["acceptance_status"], "validated")
def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None: def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None:
self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?")) self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?"))
@ -610,6 +685,32 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
self.assertIsNone(step_state["date_scope"]["as_of_date"]) self.assertIsNone(step_state["date_scope"]["as_of_date"])
self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset") self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset")
def test_out_of_window_document_dates_reject_validated_step(self) -> None:
validated = dcl.validate_step_contract(
{
"execution_status": "exact",
"required_answer_shape": "direct_answer_first",
"reply_type": "factual",
"actual_direct_answer": "Контрагент: Группа СВК. Найдено документов: 19.",
"assistant_text": (
"Контрагент: Группа СВК. Найдено документов: 19.\n"
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07"
),
"top_non_empty_lines": [
"Контрагент: Группа СВК. Найдено документов: 19.",
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07",
],
"date_scope": {
"period_from": "2020-01-01",
"period_to": "2020-12-31",
},
}
)
self.assertEqual(validated["status"], "rejected")
self.assertIn("out_of_window_date_in_answer", validated["violated_invariants"])
self.assertIn("out_of_window_date_in_answer", validated["warnings"])
def test_open_items_exact_negative_answer_validates_without_rows(self) -> None: def test_open_items_exact_negative_answer_validates_without_rows(self) -> None:
step_state = dcl.build_scenario_step_state( step_state = dcl.build_scenario_step_state(
scenario_id="open_items_negative_demo", scenario_id="open_items_negative_demo",