From 41463c9965eda8d780cee475673952349924a6f9 Mon Sep 17 00:00:00 2001 From: dctouch Date: Mon, 15 Jun 2026 22:56:22 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D0=BA=D1=80=D0=B5=D0=BF=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=B0=D0=B3=D0=B5=D0=BD=D1=82=D0=BD=D1=8B=D0=B9=20loo?= =?UTF-8?q?p=20Phase107?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../orchestration/active_domain_contract.json | 425 ++++++++++++------ docs/orchestration/detector_registry.json | 8 +- scripts/agent_detector_runner.py | 99 +++- scripts/domain_case_loop.py | 48 +- scripts/test_agent_detector_runner.py | 177 ++++++++ scripts/test_domain_case_loop_step_state.py | 101 +++++ 6 files changed, 718 insertions(+), 140 deletions(-) diff --git a/docs/orchestration/active_domain_contract.json b/docs/orchestration/active_domain_contract.json index dc32c66..1e911a9 100644 --- a/docs/orchestration/active_domain_contract.json +++ b/docs/orchestration/active_domain_contract.json @@ -1,10 +1,10 @@ { "schema_version": "domain_scenario_pack_v1", - "pack_id": "active_phase106_profit_cashflow_margin_next_step_20260615", - "domain": "address_phase106_profit_cashflow_margin_next_step", - "runtime_domain": "business_overview_profit_cashflow_margin_boundary", - "title": "Profit, cashflow, margin and next-check boundary", - "description": "Active slice for the autonomous agent loop: colloquial earnings wording must be answered as checked cashflow/accounting evidence with honest profit boundaries, then pivot safely into товарная маржа and concrete next checks.", + "pack_id": "active_phase107_mixed_planner_brain_dogfood_20260615", + "domain": "address_phase107_mixed_planner_brain_dogfood", + "runtime_domain": "planner_autonomy_mixed_boundary", + "title": "Mixed planner-brain dogfood replay", + "description": "Active slice for dogfooding the autonomous agent loop on a compact mixed planner path: living chat guard, counterparty alias grounding, follow-up money flow, net, documents, movement evidence, broad business evaluation, and off-domain recovery.", "status": "active", "source_of_truth_policy": { "purpose": "single mutable domain source for the current orchestration target", @@ -21,182 +21,349 @@ }, "analysis_context": { "as_of_date": "2026-06-15", - "source": "phase106_after_phase104_role_tail_autorun_persistence", + "source": "phase107_after_phase106_profit_cashflow_margin_next_step", "known_previous_anchors": [ - "hm_business_overview_cashflow_limits_20260602_p4", - "hm_business_profile_evaluation_limits_p3", - "agent_margin_profitability_reliability_20260615_p12_detector_scope", - "phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop" - ] + "phase83_planner_brain_alignment_mix_20260603_live9", + "phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop", + "phase106_profit_cashflow_margin_next_step_20260615_p03_loop" + ], + "why_now": "Phase83 passed in the legacy truth-harness format; this pack replays the highest-value mixed planner slice through the current autonomous pack-loop." }, "target_score": 88, "bindings": { "main_organization": "ООО Альтернатива Плюс", - "control_year": "2020" + "control_year": "2020", + "svk_counterparty": "Группа СВК" }, "issue_codes_under_test": [ - "cashflow_misreported_as_clean_profit", - "margin_domain_leak_accounting_route", + "technical_garbage_in_answer", + "business_direct_answer_missing", "business_next_step_missing", - "technical_garbage_in_answer" + "counterparty_value_flow_misrouted_to_company_profit", + "business_utility_gap" ], "detectors_under_test": [ "runtime_tokens_in_user_answer", "capability_ids_in_user_answer", + "first_line_not_direct_answer", + "top_level_scaffold_before_answer", "limited_answer_without_next_action", - "margin_domain_leak_accounting_route", - "margin_required_fields_missing", - "margin_next_action_missing" + "counterparty_value_flow_required_surface", + "counterparty_value_flow_profit_accounts_forbidden" ], "agent_audit_expectations": { "semantic_answer_review_first": true, "direct_answer_first": true, "business_utility_required": true, - "cashflow_not_clean_profit": true, - "margin_domain_purity_required": true, - "next_action_required_when_limited": true, + "selected_counterparty_memory_required": true, + "date_carryover_required": true, + "documents_and_movements_are_evidence_drilldowns": true, + "off_domain_living_chat_must_not_replay_business_context": true, "technical_garbage_forbidden": true, "do_not_accept_if": [ - "colloquial earnings wording is answered as confirmed clean profit without evidence", - "cashflow/net operating flow is silently presented as accounting profit or margin", - "margin follow-up leaks into bank payments, fixed assets, amortization, or generic accounting route", - "next-check answer is generic advice without concrete 1C evidence contours", - "final answer leaks route ids, capability ids, runtime enums, or debug payload" + "a follow-up over 'нему' loses the selected counterparty", + "a payout or net follow-up loses the 2020 period", + "a counterparty received/paid/net question is answered as company profit", + "document or movement follow-up answers with a fresh clarification instead of reusing the selected counterparty", + "broad business evaluation leaks route ids, capability ids, runtime enums, or debug payload", + "off-domain living chat is hijacked by stale 1C business context" ] }, "scenarios": [ { - "scenario_id": "profit_cashflow_margin_next_step_boundary", - "title": "Colloquial earnings, clean-profit boundary, margin pivot, and next checks", - "description": "Validate that the assistant handles colloquial earnings wording, refuses to overclaim clean profit, pivots into товарная маржа without wrong-domain leakage, and proposes concrete next 1C checks.", + "scenario_id": "mixed_planner_counterparty_evidence_and_living_guard", + "title": "Counterparty memory, evidence drilldowns, business overview, and off-domain guard", + "description": "Validate a compact but cross-cutting planner path: chat sanity, loose counterparty alias grounding, follow-up money flow and net, documents/movements evidence, explicit broad company evaluation, and recovery into ordinary living chat.", "steps": [ { - "step_id": "step_01_choose_company_scope", - "title": "Bare organization choice binds active company scope", + "step_id": "step_01_human_smalltalk_sanity", + "title": "Living chat remains human and does not expose discovery internals", "node_role": "root", - "question": "{{bindings.main_organization}}", + "question": "привет, ты на связи?", "allowed_reply_types": [ "factual_with_explanation", "partial_coverage" ], "semantic_tags": [ - "bare_org_scope", - "company_scope" - ], - "required_answer_shape": "direct_answer_first", - "required_answer_patterns_all": [ - "(?i)фиксир|рабоч.*организац|контур", - "(?i)альтернатива" - ], - "forbidden_answer_patterns": [ - "(?i)не могу определить", - "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object" - ] - }, - { - "step_id": "step_02_colloquial_earned_money_2020", - "title": "Colloquial earned-money wording stays bounded", - "question": "скока денег альтернатива заработала за 20 год?", - "depends_on": [ - "step_01_choose_company_scope" - ], - "semantic_tags": [ - "business_overview", - "colloquial_cashflow", - "cashflow_not_profit" - ], - "required_answer_shape": "direct_answer_first", - "required_answer_patterns_all": [ - "(?i)2020|20\\s*год", - "(?i)поступ|входящ|денег|заработ", - "(?i)не.*чист.*прибыл|не.*прибыл|прибыл.*не подтвержд|это.*не.*прибыл" - ], - "forbidden_answer_patterns": [ - "(?i)^\\s*Коротко:", - "(?i)это\\s+чистая\\s+прибыль", - "(?i)точно.*прибыл", - "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" - ], - "criticality": "critical" - }, - { - "step_id": "step_03_clean_profit_followup_boundary", - "title": "Clean-profit follow-up must not convert cashflow into profit", - "question": "а это чистая прибыль?", - "depends_on": [ - "step_02_colloquial_earned_money_2020" - ], - "semantic_tags": [ - "profitability_boundary", - "cashflow_not_profit", - "followup_boundary" - ], - "required_answer_shape": "direct_answer_first", - "required_answer_patterns_all": [ - "(?i)не.*чист.*прибыл|не.*является.*прибыл|не подтвержд.*прибыл|это.*не.*прибыл", - "(?i)денежн|поступ|нетто|оборот|cash|себестоим|расход|закрыва" - ], - "forbidden_answer_patterns": [ - "(?i)^\\s*Коротко:", - "(?i)^\\s*да[,\\s]+это.*прибыл", - "(?i)точно.*чист.*прибыл", - "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" - ], - "criticality": "critical" - }, - { - "step_id": "step_04_margin_pivot_keeps_inventory_domain", - "title": "Margin pivot uses inventory margin contour, not payments or fixed assets", - "question": "тогда какую прибыль или маржу можно проверить по товарам за 2020?", - "depends_on": [ - "step_03_clean_profit_followup_boundary" - ], - "expected_intents": [ - "inventory_margin_ranking_for_nomenclature" - ], - "expected_capability": "inventory_inventory_margin_ranking_for_nomenclature", - "expected_recipe": "address_inventory_margin_ranking_for_nomenclature_v1", - "semantic_tags": [ - "margin_profitability", - "inventory", - "wrong_domain_trap", - "cashflow_to_margin_pivot" + "human_answer", + "mcp_discovery_gate_sanity", + "meta_smalltalk" ], "required_answer_shape": "direct_answer_first", "required_answer_patterns_any": [ - "(?i)марж|валов|выруч|себестоим|номенклатур|товар|не могу подтвердить|не хватает" + "(?i)привет|на связи|готов|помочь" ], "forbidden_answer_patterns": [ - "(?i)амортизац|основн(ые|ых)? средств|объект ОС|банк|оплат[аы]|payment_document|settlement", + "(?i)mcp|runtime_|query_documents|primitive|route_candidate|capability_id|snapshot_items|answer_object" + ], + "criticality": "info" + }, + { + "step_id": "step_02_resolve_counterparty_alias", + "title": "Entity resolution grounds the checked 1C counterparty from a loose alias", + "question": "найди в 1С контрагента СВК", + "depends_on": [ + "step_01_human_smalltalk_sanity" + ], + "allowed_reply_types": [ + "factual", + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "entity_resolution", + "alias_grounding", + "followup_anchor", + "planner_catalog_alignment" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)свк", + "(?i)контрагент" + ], + "required_answer_patterns_any": [ + "(?i)группа\\s+свк", + "(?i)каталог|найден|наиболее вероятн" + ], + "forbidden_answer_patterns": [ + "(?i)получили|заплатили|нетто|оборот|выручк|сумм(а|ы)", "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" ], "criticality": "critical" }, { - "step_id": "step_05_next_checks_are_concrete_1c_contours", - "title": "Next checks must be concrete 1C evidence contours", - "question": "что дальше проверить, чтобы понять здоровье бизнеса, без гаданий?", + "step_id": "step_03_incoming_by_resolved_entity", + "title": "Incoming value-flow follow-up reuses the resolved counterparty anchor", + "question": "сколько получили по нему за 2020 год", "depends_on": [ - "step_04_margin_pivot_keeps_inventory_domain" + "step_02_resolve_counterparty_alias" + ], + "allowed_reply_types": [ + "factual_with_explanation", + "partial_coverage" ], "semantic_tags": [ - "business_next_step", - "evidence_plan", - "no_generic_advice" + "entity_resolution", + "incoming_value_flow", + "followup_reuse", + "date_carryover", + "planner_catalog_alignment" ], "required_answer_shape": "direct_answer_first", "required_answer_patterns_all": [ - "(?i)прибыл|марж|себестоим|финрезульт", - "(?i)ндс|налог", - "(?i)дебитор|кредитор|долг|расчет", - "(?i)склад|остат|оборач|товар" + "(?i)2020", + "(?i)получил|входящ|поступ", + "(?i)руб" + ], + "required_answer_patterns_any": [ + "(?i)группа\\s+свк", + "(?i)свк" ], "forbidden_answer_patterns": [ - "(?i)^\\s*Коротко:", - "(?i)просто.*посмотр|общ.*рекоменд|улучшить маркетинг|нанять|стратег", + "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту", + "(?i)чист.*прибыл|90/91/99|финрезульт", "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" ], - "criticality": "important" + "criticality": "critical" + }, + { + "step_id": "step_04_payout_switch_by_resolved_entity", + "title": "Outgoing payment follow-up keeps the same grounded counterparty and checked year", + "question": "а теперь сколько заплатили?", + "depends_on": [ + "step_03_incoming_by_resolved_entity" + ], + "allowed_reply_types": [ + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "entity_resolution", + "payout_switch", + "followup_reuse", + "date_carryover", + "planner_catalog_alignment" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)2020", + "(?i)заплатил|исходящ|списан|платеж|платёж", + "(?i)руб" + ], + "required_answer_patterns_any": [ + "(?i)группа\\s+свк", + "(?i)свк" + ], + "forbidden_answer_patterns": [ + "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год", + "(?i)чист.*прибыл|90/91/99|финрезульт", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "critical" + }, + { + "step_id": "step_05_net_after_payout", + "title": "Net-flow follow-up reuses the same grounded counterparty and checked year after payout", + "question": "а какое нетто?", + "depends_on": [ + "step_04_payout_switch_by_resolved_entity" + ], + "allowed_reply_types": [ + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "entity_resolution", + "net_value_flow", + "followup_reuse", + "date_carryover", + "planner_catalog_alignment" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)2020", + "(?i)нетто|сальдо|разниц", + "(?i)получ", + "(?i)заплат", + "(?i)руб" + ], + "required_answer_patterns_any": [ + "12[ .\\u00a0]?093[ .\\u00a0]?465|12093465", + "(?i)группа\\s+свк|свк" + ], + "forbidden_answer_patterns": [ + "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год", + "(?i)чист.*прибыл|90/91/99|финрезульт", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "critical" + }, + { + "step_id": "step_06_documents_after_net", + "title": "Document evidence follow-up keeps the grounded counterparty after the net answer", + "question": "а по документам?", + "depends_on": [ + "step_05_net_after_payout" + ], + "allowed_reply_types": [ + "factual", + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "entity_resolution", + "document_evidence", + "value_flow_pivot", + "followup_reuse", + "planner_catalog_alignment" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)документ|счет|счёт|накладн|акт" + ], + "required_answer_patterns_any": [ + "(?i)группа\\s+свк", + "(?i)свк", + "(?i)2020" + ], + "forbidden_answer_patterns": [ + "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту", + "(?i)сколько получили|сколько заплатили|нетто", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "critical" + }, + { + "step_id": "step_07_movements_after_documents", + "title": "Movement evidence follow-up keeps the grounded counterparty after the document answer", + "question": "а по движениям?", + "depends_on": [ + "step_06_documents_after_net" + ], + "allowed_reply_types": [ + "factual", + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "entity_resolution", + "movement_evidence", + "document_pivot", + "followup_reuse", + "planner_catalog_alignment" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)движени|операц|платеж|платёж|списан|поступ" + ], + "required_answer_patterns_any": [ + "(?i)группа\\s+свк", + "(?i)свк", + "(?i)2020" + ], + "forbidden_answer_patterns": [ + "(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту", + "(?i)сколько получили|сколько заплатили|нетто", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "critical" + }, + { + "step_id": "step_08_broad_business_evaluation_explicit_org", + "title": "Broad business evaluation stays grounded and bounded after counterparty drilldowns", + "question": "Как ты оценишь деятельность компании {{bindings.main_organization}} по данным 1С?", + "depends_on": [ + "step_07_movements_after_documents" + ], + "allowed_reply_types": [ + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "broad_business_evaluation", + "grounded_summary", + "context_switch_from_counterparty_to_company" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_all": [ + "(?i)альтернатива", + "(?i)ограниченн|проверенн|не\\s+аудит|не\\s+аудитор", + "(?i)1с|подтвержд", + "(?i)денежн|долг|ндс|контрагент|операц" + ], + "forbidden_answer_patterns": [ + "(?i)^\\s*коротко:", + "(?i)чистая\\s+прибыль\\s+подтверждена|аудиторское\\s+заключение", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "critical" + }, + { + "step_id": "step_09_off_domain_living_chat_not_hijacked", + "title": "Off-domain living chat is not hijacked by stale business context", + "question": "а чем капибара отличается от утки?", + "depends_on": [ + "step_08_broad_business_evaluation_explicit_org" + ], + "allowed_reply_types": [ + "factual_with_explanation", + "partial_coverage" + ], + "semantic_tags": [ + "off_domain_living_chat", + "stale_replay_forbidden", + "context_boundary" + ], + "required_answer_shape": "direct_answer_first", + "required_answer_patterns_any": [ + "(?i)капибар.*утк|утк.*капибар", + "(?i)млекопита|птиц|грызун" + ], + "forbidden_answer_patterns": [ + "(?i)альтернатива|свк|контрагент|1с|поступлен|платеж|платёж|нетто", + "(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery" + ], + "criticality": "warning" } ] } diff --git a/docs/orchestration/detector_registry.json b/docs/orchestration/detector_registry.json index 845694a..e1498b3 100644 --- a/docs/orchestration/detector_registry.json +++ b/docs/orchestration/detector_registry.json @@ -218,7 +218,9 @@ "issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"], "inputs": ["steps//output.md"], "check": { - "artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"], + "artifact_path_include_patterns": [ + "(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_05_net_after_payout)" + ], "required_patterns_any": [ "(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))" ] @@ -231,7 +233,9 @@ "issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"], "inputs": ["steps//output.md"], "check": { - "artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"], + "artifact_path_include_patterns": [ + "(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_03_incoming_by_resolved_entity|step_04_payout_switch_by_resolved_entity|step_05_net_after_payout)" + ], "forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"] } }, diff --git a/scripts/agent_detector_runner.py b/scripts/agent_detector_runner.py index 4165a74..641d9a3 100644 --- a/scripts/agent_detector_runner.py +++ b/scripts/agent_detector_runner.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse import json +import os import re from datetime import datetime, timezone from pathlib import Path @@ -18,8 +19,35 @@ DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS = [ ] +def path_for_io(path: Path) -> str: + if os.name != "nt": + return str(path) + raw_value = str(path) + if raw_value.startswith("\\\\?\\"): + return raw_value + absolute_path = path if path.is_absolute() else Path.cwd() / path + absolute_value = str(absolute_path.absolute()) + if absolute_value.startswith("\\\\?\\"): + return absolute_value + if absolute_value.startswith("\\\\"): + return "\\\\?\\UNC\\" + absolute_value.lstrip("\\") + return "\\\\?\\" + absolute_value + + +def path_exists(path: Path) -> bool: + try: + return os.path.exists(path_for_io(path)) + except OSError: + return False + + +def read_text(path: Path) -> str: + with open(path_for_io(path), "r", encoding="utf-8") as handle: + return handle.read() + + def read_json(path: Path) -> Any: - return json.loads(path.read_text(encoding="utf-8")) + return json.loads(read_text(path)) def read_json_object(path: Path) -> dict[str, Any]: @@ -31,8 +59,9 @@ def read_json_object(path: Path) -> dict[str, Any]: def write_json(path: Path, payload: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + os.makedirs(path_for_io(path.parent), exist_ok=True) + with open(path_for_io(path), "w", encoding="utf-8") as handle: + handle.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n") def utc_now() -> str: @@ -164,7 +193,7 @@ def select_detectors( def read_text_or_empty(path: Path) -> str: try: - return path.read_text(encoding="utf-8") + return read_text(path) except OSError: return "" @@ -207,7 +236,7 @@ def first_pattern_search(patterns: list[re.Pattern[str]], text: str) -> re.Match def assistant_text_from_turn_path(path: Path | None) -> str: - if path is None or not path.exists(): + if path is None or not path_exists(path): return "" payload = read_json_object(path) assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {} @@ -264,14 +293,21 @@ def output_turn_path(output_path: Path) -> Path | None: name = output_path.name if name == "output.md": candidate = output_path.with_name("turn.json") - return candidate if candidate.exists() else None + return candidate if path_exists(candidate) else None if name.endswith("_output.md"): prefix = name[: -len("_output.md")] candidate = output_path.with_name(f"{prefix}_turn.json") - return candidate if candidate.exists() else None + return candidate if path_exists(candidate) else None return None +def output_step_state_path(output_path: Path) -> Path | None: + if output_path.name != "output.md": + return None + candidate = output_path.with_name("step_state.json") + return candidate if path_exists(candidate) else None + + def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]: outputs: list[dict[str, Any]] = [] seen: set[Path] = set() @@ -292,6 +328,7 @@ def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]: "artifact_path": str(path.relative_to(artifact_dir)), "text": read_text_or_empty(path), "turn_path": turn_path, + "step_state_path": output_step_state_path(path), } ) return outputs @@ -497,6 +534,52 @@ def evaluate_limited_next_action( return build_result(detector_name, detector, status, message, evidence=failures) +def evaluate_answer_text_shape( + detector_name: str, + detector: dict[str, Any], + outputs: list[dict[str, Any]], +) -> dict[str, Any]: + if not outputs: + return build_result(detector_name, detector, "skipped", "no output.md-style artifacts matched detector scope") + failures: list[dict[str, Any]] = [] + reviewed: list[dict[str, Any]] = [] + unknown: list[dict[str, Any]] = [] + for output in outputs: + step_state_path = output.get("step_state_path") + step_state = read_json_object(step_state_path) if isinstance(step_state_path, Path) else {} + review = step_state.get("business_first_review") if isinstance(step_state.get("business_first_review"), dict) else {} + direct_answer_first_ok = review.get("direct_answer_first_ok") + if direct_answer_first_ok is True: + reviewed.append({"path": output["repo_path"], "direct_answer_first_ok": True}) + elif direct_answer_first_ok is False: + failures.append({"path": output["repo_path"], "direct_answer_first_ok": False}) + else: + unknown.append({"path": output["repo_path"], "reason": "business_first_review_missing"}) + if failures: + return build_result( + detector_name, + detector, + "fail", + "first-line direct answer check failed", + evidence=[*failures, *unknown], + ) + if unknown: + return build_result( + detector_name, + detector, + "review", + "direct-answer shape requires business review", + evidence=[*reviewed, *unknown], + ) + return build_result( + detector_name, + detector, + "pass", + "business-first step reviews confirm direct-answer-first shape", + evidence=reviewed, + ) + + def evaluate_trace_guard( detector_name: str, detector: dict[str, Any], @@ -686,7 +769,7 @@ def evaluate_detector( ): return evaluate_composite(detector_name, detector, results_by_name) if kind == "answer_text_shape": - return evaluate_manual_review(detector_name, detector, "direct-answer shape requires business review") + return evaluate_answer_text_shape(detector_name, detector, scoped_outputs) return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}") diff --git a/scripts/domain_case_loop.py b/scripts/domain_case_loop.py index a424ff2..bb0fcfa 100644 --- a/scripts/domain_case_loop.py +++ b/scripts/domain_case_loop.py @@ -317,6 +317,7 @@ DEFAULT_INVARIANT_SEVERITY: dict[str, str] = { "forbidden_recipe_selected": "P0", "focus_object_missing": "P0", "wrong_date_scope_state": "P0", + "out_of_window_date_in_answer": "P0", "direct_answer_missing": "P0", "top_level_noise_present": "P0", "business_direct_answer_missing": "P0", @@ -930,6 +931,40 @@ def first_non_empty_lines(text: str, limit: int = 3) -> list[str]: return output +def expected_single_year_from_date_scope(date_scope: Any) -> str | None: + if not isinstance(date_scope, dict): + return None + if str(date_scope.get("scope") or "").strip().lower() == "all_time": + return None + period_from = normalize_iso_date(date_scope.get("period_from")) + period_to = normalize_iso_date(date_scope.get("period_to")) + if not period_from or not period_to: + return None + from_year = period_from[:4] + to_year = period_to[:4] + if from_year != to_year: + return None + return from_year if re.fullmatch(r"(?:19|20)\d{2}", from_year) else None + + +def dated_evidence_years_from_answer(text: Any) -> list[str]: + source = repair_text_mojibake(str(text or "")) + years: list[str] = [] + for match in re.finditer(r"\b((?:19|20)\d{2})-\d{2}-\d{2}(?:T|\b)", source): + years.append(match.group(1)) + for match in re.finditer(r"\b\d{1,2}\.\d{1,2}\.((?:19|20)\d{2})\b", source): + years.append(match.group(1)) + return list(dict.fromkeys(years)) + + +def answer_has_out_of_window_dates_for_scope(text: Any, date_scope: Any) -> bool: + expected_year = expected_single_year_from_date_scope(date_scope) + if not expected_year: + return False + years = dated_evidence_years_from_answer(text) + return any(year != expected_year for year in years) + + def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]: scenario_tree = raw_contract.get("scenario_tree") if not isinstance(scenario_tree, dict): @@ -2867,7 +2902,12 @@ def is_validated_clean_meta_chat_answer( return False semantic_tags = set(normalize_string_list(state.get("semantic_tags"))) allowed_tags = { + "human_answer", "meta_smalltalk", + "mcp_discovery_gate_sanity", + "off_domain_living_chat", + "stale_replay_forbidden", + "context_boundary", "company_selected", "organization_authority", "meta_capability", @@ -3112,6 +3152,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]: if current_date_scope and current_date_scope != required_filters["as_of_date"]: violated_invariants.append("wrong_date_scope_state") + if answer_has_out_of_window_dates_for_scope(assistant_text, date_scope): + violated_invariants.append("out_of_window_date_in_answer") + warnings.append("out_of_window_date_in_answer") + if should_require_direct_answer(state): if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer): violated_invariants.append("direct_answer_missing") @@ -3193,8 +3237,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]: state["clarification_answer_validated"] = clarification_validated state["missing_axis_clarification_validated"] = missing_axis_clarification_validated state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated + effective_execution_status = "exact" if clean_meta_chat_validated else execution_status + state["execution_status"] = effective_execution_status state["acceptance_status"] = acceptance_status_from_execution( - execution_status, + effective_execution_status, hard_fail, ( bounded_validated diff --git a/scripts/test_agent_detector_runner.py b/scripts/test_agent_detector_runner.py index 7fdcb80..657a117 100644 --- a/scripts/test_agent_detector_runner.py +++ b/scripts/test_agent_detector_runner.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import sys import tempfile import unittest @@ -287,6 +288,182 @@ class AgentDetectorRunnerTests(unittest.TestCase): self.assertEqual(len(evidence_paths), 1) self.assertIn("step_01_margin_root", evidence_paths[0]) + def test_counterparty_value_flow_required_surface_scopes_to_current_net_step(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact_dir = root / "run" + write_text( + artifact_dir + / "scenarios" + / "mixed_planner_counterparty_evidence_and_living_guard" + / "steps" + / "step_03_incoming_by_resolved_entity" + / "output.md", + "Входящие денежные поступления по контрагенту Группа СВК за 2020: 12 093 465 руб.", + ) + write_text( + artifact_dir + / "scenarios" + / "mixed_planner_counterparty_evidence_and_living_guard" + / "steps" + / "step_05_net_after_payout" + / "output.md", + "По контрагенту Группа СВК за период 2020 получили 12 093 465 руб., заплатили 0 руб.; расчетное нетто в нашу сторону: 12 093 465 руб.", + ) + registry_path = root / "detector_registry.json" + issue_catalog_path = root / "issue_catalog.json" + write_json( + registry_path, + { + "schema_version": "agent_detector_registry_v1", + "detectors": { + "counterparty_value_flow_required_surface": { + "kind": "answer_text_required_any", + "automation_level": "automatic", + "description": "Net value-flow answer must surface all money directions.", + "issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"], + "inputs": ["output.md"], + "check": { + "artifact_path_include_patterns": ["(?i)step_05_net_after_payout"], + "required_patterns_any": [ + "(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))" + ], + }, + } + }, + }, + ) + write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}}) + + results = runner.build_detector_results( + artifact_dir, + detector_names=["counterparty_value_flow_required_surface"], + registry_path=registry_path, + issue_catalog_path=issue_catalog_path, + include_default_global=False, + ) + + self.assertEqual(results["summary"]["status"], "pass") + self.assertEqual(results["results"][0]["status"], "pass") + evidence_paths = [item["path"] for item in results["results"][0]["evidence"]] + self.assertEqual(len(evidence_paths), 1) + self.assertIn("step_05_net_after_payout", evidence_paths[0]) + + def test_answer_text_shape_uses_business_first_step_review(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact_dir = root / "run" + step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01" + write_text(step_dir / "output.md", "Direct business answer first.") + write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": True}}) + registry_path = root / "detector_registry.json" + issue_catalog_path = root / "issue_catalog.json" + write_json( + registry_path, + { + "schema_version": "agent_detector_registry_v1", + "detectors": { + "first_line_not_direct_answer": { + "kind": "answer_text_shape", + "automation_level": "semi_automatic", + "description": "First line should be direct.", + "issue_codes": ["business_direct_answer_missing"], + "inputs": ["output.md"], + "check": {"first_line_should_be": "business_answer_or_honest_boundary"}, + } + }, + }, + ) + write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}}) + + results = runner.build_detector_results( + artifact_dir, + detector_names=["first_line_not_direct_answer"], + registry_path=registry_path, + issue_catalog_path=issue_catalog_path, + include_default_global=False, + ) + + self.assertEqual(results["summary"]["status"], "pass") + self.assertEqual(results["results"][0]["status"], "pass") + + def test_answer_text_shape_fails_when_business_first_review_rejects_first_line(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact_dir = root / "run" + step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01" + write_text(step_dir / "output.md", "Let me inspect the route first.") + write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": False}}) + registry_path = root / "detector_registry.json" + issue_catalog_path = root / "issue_catalog.json" + write_json( + registry_path, + { + "schema_version": "agent_detector_registry_v1", + "detectors": { + "first_line_not_direct_answer": { + "kind": "answer_text_shape", + "automation_level": "semi_automatic", + "description": "First line should be direct.", + "issue_codes": ["business_direct_answer_missing"], + "inputs": ["output.md"], + "check": {"first_line_should_be": "business_answer_or_honest_boundary"}, + } + }, + }, + ) + write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}}) + + results = runner.build_detector_results( + artifact_dir, + detector_names=["first_line_not_direct_answer"], + registry_path=registry_path, + issue_catalog_path=issue_catalog_path, + include_default_global=False, + ) + + self.assertEqual(results["summary"]["status"], "fail") + self.assertEqual(results["results"][0]["status"], "fail") + + def test_step_state_lookup_handles_long_artifact_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + step_dir = ( + root + / "run" + / "scenarios" + / ("mixed_planner_counterparty_evidence_and_living_guard_" + "s" * 48) + / "steps" + / ("step_04_payout_switch_by_resolved_entity_" + "x" * 80) + ) + output_path = step_dir / "output.md" + step_state_path = step_dir / "step_state.json" + try: + os.makedirs(runner.path_for_io(step_dir), exist_ok=True) + with open(runner.path_for_io(output_path), "w", encoding="utf-8") as handle: + handle.write("Direct business answer first.") + runner.write_json(step_state_path, {"business_first_review": {"direct_answer_first_ok": True}}) + + self.assertGreater(len(str(step_state_path.absolute())), 260) + self.assertEqual(runner.output_step_state_path(output_path), step_state_path) + self.assertEqual( + runner.read_json_object(step_state_path).get("business_first_review"), + {"direct_answer_first_ok": True}, + ) + finally: + for path in (output_path, step_state_path): + try: + os.remove(runner.path_for_io(path)) + except OSError: + pass + current = step_dir + while current != root: + try: + os.rmdir(runner.path_for_io(current)) + except OSError: + pass + current = current.parent + def test_composite_detector_fails_after_child_detector_fails(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/scripts/test_domain_case_loop_step_state.py b/scripts/test_domain_case_loop_step_state.py index cd6e06a..369db67 100644 --- a/scripts/test_domain_case_loop_step_state.py +++ b/scripts/test_domain_case_loop_step_state.py @@ -153,6 +153,81 @@ class DomainCaseLoopStepStateTests(unittest.TestCase): self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial") self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted") + def test_clean_meta_smalltalk_is_exact_without_1c_capability(self) -> None: + step_state = dcl.build_scenario_step_state( + scenario_id="meta_chat_demo", + domain="agentic_loop", + step={ + "step_id": "step_01", + "title": "Human smalltalk", + "depends_on": [], + "question_template": "привет, ты на связи?", + "semantic_tags": ["human_answer", "meta_smalltalk", "mcp_discovery_gate_sanity"], + "required_answer_shape": "direct_answer_first", + }, + step_index=1, + question_resolved="привет, ты на связи?", + analysis_context={}, + turn_artifact={ + "assistant_message": { + "reply_type": "factual_with_explanation", + "text": "Привет! Да, я на связи. Готов помочь с анализом данных из 1С в режиме чтения.", + "message_id": "msg-1", + "trace_id": "trace-1", + }, + "technical_debug_payload": { + "detected_mode": "chat", + "fallback_type": "none", + "living_chat_response_source": "llm_chat", + }, + "session_summary": {}, + }, + entries=[], + ) + + self.assertTrue(step_state["clean_meta_chat_answer_validated"]) + self.assertEqual(step_state["execution_status"], "exact") + self.assertEqual(step_state["acceptance_status"], "validated") + + def test_off_domain_living_chat_is_exact_without_1c_capability(self) -> None: + step_state = dcl.build_scenario_step_state( + scenario_id="off_domain_demo", + domain="agentic_loop", + step={ + "step_id": "step_09", + "title": "Off-domain living chat", + "depends_on": [], + "question_template": "а чем капибара отличается от утки?", + "semantic_tags": ["off_domain_living_chat", "stale_replay_forbidden", "context_boundary"], + "required_answer_shape": "direct_answer_first", + }, + step_index=9, + question_resolved="а чем капибара отличается от утки?", + analysis_context={}, + turn_artifact={ + "assistant_message": { + "reply_type": "factual_with_explanation", + "text": ( + "Капибара и утка отличаются принципиально: капибара - млекопитающее-грызун, " + "а утка - птица. Поэтому у них разные тело, среда обитания и способ передвижения." + ), + "message_id": "msg-2", + "trace_id": "trace-2", + }, + "technical_debug_payload": { + "detected_mode": "chat", + "fallback_type": "none", + "living_chat_response_source": "llm_chat", + }, + "session_summary": {}, + }, + entries=[], + ) + + self.assertTrue(step_state["clean_meta_chat_answer_validated"]) + self.assertEqual(step_state["execution_status"], "exact") + self.assertEqual(step_state["acceptance_status"], "validated") + def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None: self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?")) @@ -610,6 +685,32 @@ class DomainCaseLoopStepStateTests(unittest.TestCase): self.assertIsNone(step_state["date_scope"]["as_of_date"]) self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset") + def test_out_of_window_document_dates_reject_validated_step(self) -> None: + validated = dcl.validate_step_contract( + { + "execution_status": "exact", + "required_answer_shape": "direct_answer_first", + "reply_type": "factual", + "actual_direct_answer": "Контрагент: Группа СВК. Найдено документов: 19.", + "assistant_text": ( + "Контрагент: Группа СВК. Найдено документов: 19.\n" + "1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07" + ), + "top_non_empty_lines": [ + "Контрагент: Группа СВК. Найдено документов: 19.", + "1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07", + ], + "date_scope": { + "period_from": "2020-01-01", + "period_to": "2020-12-31", + }, + } + ) + + self.assertEqual(validated["status"], "rejected") + self.assertIn("out_of_window_date_in_answer", validated["violated_invariants"]) + self.assertIn("out_of_window_date_in_answer", validated["warnings"]) + def test_open_items_exact_negative_answer_validates_without_rows(self) -> None: step_state = dcl.build_scenario_step_state( scenario_id="open_items_negative_demo",