SILVER
/
NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Укрепить агентный loop Phase107

This commit is contained in:
dctouch 2026-06-15 22:56:22 +03:00
parent b3f2e405bb
commit 41463c9965
6 changed files with 718 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

425
docs/orchestration/active_domain_contract.json
View File

@ -1,10 +1,10 @@
{
"schema_version": "domain_scenario_pack_v1",
"pack_id": "active_phase106_profit_cashflow_margin_next_step_20260615",
"domain": "address_phase106_profit_cashflow_margin_next_step",
"runtime_domain": "business_overview_profit_cashflow_margin_boundary",
"title": "Profit, cashflow, margin and next-check boundary",
"description": "Active slice for the autonomous agent loop: colloquial earnings wording must be answered as checked cashflow/accounting evidence with honest profit boundaries, then pivot safely into товарная маржа and concrete next checks.",
"pack_id": "active_phase107_mixed_planner_brain_dogfood_20260615",
"domain": "address_phase107_mixed_planner_brain_dogfood",
"runtime_domain": "planner_autonomy_mixed_boundary",
"title": "Mixed planner-brain dogfood replay",
"description": "Active slice for dogfooding the autonomous agent loop on a compact mixed planner path: living chat guard, counterparty alias grounding, follow-up money flow, net, documents, movement evidence, broad business evaluation, and off-domain recovery.",
"status": "active",
"source_of_truth_policy": {
"purpose": "single mutable domain source for the current orchestration target",
@ -21,182 +21,349 @@
},
"analysis_context": {
"as_of_date": "2026-06-15",
"source": "phase106_after_phase104_role_tail_autorun_persistence",
"source": "phase107_after_phase106_profit_cashflow_margin_next_step",
"known_previous_anchors": [
"hm_business_overview_cashflow_limits_20260602_p4",
"hm_business_profile_evaluation_limits_p3",
"agent_margin_profitability_reliability_20260615_p12_detector_scope",
"phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop"
]
"phase83_planner_brain_alignment_mix_20260603_live9",
"phase104_generic_role_tail_anchor_hygiene_20260615_p04_loop",
"phase106_profit_cashflow_margin_next_step_20260615_p03_loop"
],
"why_now": "Phase83 passed in the legacy truth-harness format; this pack replays the highest-value mixed planner slice through the current autonomous pack-loop."
},
"target_score": 88,
"bindings": {
"main_organization": "ООО Альтернатива Плюс",
"control_year": "2020"
"control_year": "2020",
"svk_counterparty": "Группа СВК"
},
"issue_codes_under_test": [
"cashflow_misreported_as_clean_profit",
"margin_domain_leak_accounting_route",
"technical_garbage_in_answer",
"business_direct_answer_missing",
"business_next_step_missing",
"technical_garbage_in_answer"
"counterparty_value_flow_misrouted_to_company_profit",
"business_utility_gap"
],
"detectors_under_test": [
"runtime_tokens_in_user_answer",
"capability_ids_in_user_answer",
"first_line_not_direct_answer",
"top_level_scaffold_before_answer",
"limited_answer_without_next_action",
"margin_domain_leak_accounting_route",
"margin_required_fields_missing",
"margin_next_action_missing"
"counterparty_value_flow_required_surface",
"counterparty_value_flow_profit_accounts_forbidden"
],
"agent_audit_expectations": {
"semantic_answer_review_first": true,
"direct_answer_first": true,
"business_utility_required": true,
"cashflow_not_clean_profit": true,
"margin_domain_purity_required": true,
"next_action_required_when_limited": true,
"selected_counterparty_memory_required": true,
"date_carryover_required": true,
"documents_and_movements_are_evidence_drilldowns": true,
"off_domain_living_chat_must_not_replay_business_context": true,
"technical_garbage_forbidden": true,
"do_not_accept_if": [
"colloquial earnings wording is answered as confirmed clean profit without evidence",
"cashflow/net operating flow is silently presented as accounting profit or margin",
"margin follow-up leaks into bank payments, fixed assets, amortization, or generic accounting route",
"next-check answer is generic advice without concrete 1C evidence contours",
"final answer leaks route ids, capability ids, runtime enums, or debug payload"
"a follow-up over 'нему' loses the selected counterparty",
"a payout or net follow-up loses the 2020 period",
"a counterparty received/paid/net question is answered as company profit",
"document or movement follow-up answers with a fresh clarification instead of reusing the selected counterparty",
"broad business evaluation leaks route ids, capability ids, runtime enums, or debug payload",
"off-domain living chat is hijacked by stale 1C business context"
]
},
"scenarios": [
{
"scenario_id": "profit_cashflow_margin_next_step_boundary",
"title": "Colloquial earnings, clean-profit boundary, margin pivot, and next checks",
"description": "Validate that the assistant handles colloquial earnings wording, refuses to overclaim clean profit, pivots into товарная маржа without wrong-domain leakage, and proposes concrete next 1C checks.",
"scenario_id": "mixed_planner_counterparty_evidence_and_living_guard",
"title": "Counterparty memory, evidence drilldowns, business overview, and off-domain guard",
"description": "Validate a compact but cross-cutting planner path: chat sanity, loose counterparty alias grounding, follow-up money flow and net, documents/movements evidence, explicit broad company evaluation, and recovery into ordinary living chat.",
"steps": [
{
"step_id": "step_01_choose_company_scope",
"title": "Bare organization choice binds active company scope",
"step_id": "step_01_human_smalltalk_sanity",
"title": "Living chat remains human and does not expose discovery internals",
"node_role": "root",
"question": "{{bindings.main_organization}}",
"question": "привет, ты на связи?",
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"bare_org_scope",
"company_scope"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)фиксир|рабоч.*организац|контур",
"(?i)альтернатива"
],
"forbidden_answer_patterns": [
"(?i)не могу определить",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object"
]
},
{
"step_id": "step_02_colloquial_earned_money_2020",
"title": "Colloquial earned-money wording stays bounded",
"question": "скока денег альтернатива заработала за 20 год?",
"depends_on": [
"step_01_choose_company_scope"
],
"semantic_tags": [
"business_overview",
"colloquial_cashflow",
"cashflow_not_profit"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020|20\\s*год",
"(?i)поступ|входящ|денег|заработ",
"(?i)не.*чист.*прибыл|не.*прибыл|прибыл.*не подтвержд|это.*не.*прибыл"
],
"forbidden_answer_patterns": [
"(?i)^\\s*Коротко:",
"(?i)это\\s+чистая\\s+прибыль",
"(?i)точно.*прибыл",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_03_clean_profit_followup_boundary",
"title": "Clean-profit follow-up must not convert cashflow into profit",
"question": "а это чистая прибыль?",
"depends_on": [
"step_02_colloquial_earned_money_2020"
],
"semantic_tags": [
"profitability_boundary",
"cashflow_not_profit",
"followup_boundary"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)не.*чист.*прибыл|не.*является.*прибыл|не подтвержд.*прибыл|это.*не.*прибыл",
"(?i)денежн|поступ|нетто|оборот|cash|себестоим|расход|закрыва"
],
"forbidden_answer_patterns": [
"(?i)^\\s*Коротко:",
"(?i)^\\s*да[,\\s]+это.*прибыл",
"(?i)точно.*чист.*прибыл",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_04_margin_pivot_keeps_inventory_domain",
"title": "Margin pivot uses inventory margin contour, not payments or fixed assets",
"question": "тогда какую прибыль или маржу можно проверить по товарам за 2020?",
"depends_on": [
"step_03_clean_profit_followup_boundary"
],
"expected_intents": [
"inventory_margin_ranking_for_nomenclature"
],
"expected_capability": "inventory_inventory_margin_ranking_for_nomenclature",
"expected_recipe": "address_inventory_margin_ranking_for_nomenclature_v1",
"semantic_tags": [
"margin_profitability",
"inventory",
"wrong_domain_trap",
"cashflow_to_margin_pivot"
"human_answer",
"mcp_discovery_gate_sanity",
"meta_smalltalk"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_any": [
"(?i)марж|валов|выруч|себестоим|номенклатур|товар|не могу подтвердить|не хватает"
"(?i)привет|на связи|готов|помочь"
],
"forbidden_answer_patterns": [
"(?i)амортизац|основн(ые|ых)? средств|объект ОС|банк|оплат[аы]|payment_document|settlement",
"(?i)mcp|runtime_|query_documents|primitive|route_candidate|capability_id|snapshot_items|answer_object"
],
"criticality": "info"
},
{
"step_id": "step_02_resolve_counterparty_alias",
"title": "Entity resolution grounds the checked 1C counterparty from a loose alias",
"question": "найди в 1С контрагента СВК",
"depends_on": [
"step_01_human_smalltalk_sanity"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"alias_grounding",
"followup_anchor",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)свк",
"(?i)контрагент"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)каталог|найден|наиболее вероятн"
],
"forbidden_answer_patterns": [
"(?i)получили|заплатили|нетто|оборот|выручк|сумм(а|ы)",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_05_next_checks_are_concrete_1c_contours",
"title": "Next checks must be concrete 1C evidence contours",
"question": "что дальше проверить, чтобы понять здоровье бизнеса, без гаданий?",
"step_id": "step_03_incoming_by_resolved_entity",
"title": "Incoming value-flow follow-up reuses the resolved counterparty anchor",
"question": "сколько получили по нему за 2020 год",
"depends_on": [
"step_04_margin_pivot_keeps_inventory_domain"
"step_02_resolve_counterparty_alias"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"business_next_step",
"evidence_plan",
"no_generic_advice"
"entity_resolution",
"incoming_value_flow",
"followup_reuse",
"date_carryover",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)прибыл|марж|себестоим|финрезульт",
"(?i)ндс|налог",
"(?i)дебитор|кредитор|долг|расчет",
"(?i)склад|остат|оборач|товар"
"(?i)2020",
"(?i)получил|входящ|поступ",
"(?i)руб"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк"
],
"forbidden_answer_patterns": [
"(?i)^\\s*Коротко:",
"(?i)просто.*посмотр|общ.*рекоменд|улучшить маркетинг|нанять|стратег",
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "important"
"criticality": "critical"
},
{
"step_id": "step_04_payout_switch_by_resolved_entity",
"title": "Outgoing payment follow-up keeps the same grounded counterparty and checked year",
"question": "а теперь сколько заплатили?",
"depends_on": [
"step_03_incoming_by_resolved_entity"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"payout_switch",
"followup_reuse",
"date_carryover",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020",
"(?i)заплатил|исходящ|списан|платеж|платёж",
"(?i)руб"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
"(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_05_net_after_payout",
"title": "Net-flow follow-up reuses the same grounded counterparty and checked year after payout",
"question": "а какое нетто?",
"depends_on": [
"step_04_payout_switch_by_resolved_entity"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"net_value_flow",
"followup_reuse",
"date_carryover",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)2020",
"(?i)нетто|сальдо|разниц",
"(?i)получ",
"(?i)заплат",
"(?i)руб"
],
"required_answer_patterns_any": [
"12[ .\\u00a0]?093[ .\\u00a0]?465|12093465",
"(?i)группа\\s+свк|свк"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту|за какой год",
"(?i)чист.*прибыл|90/91/99|финрезульт",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_06_documents_after_net",
"title": "Document evidence follow-up keeps the grounded counterparty after the net answer",
"question": "а по документам?",
"depends_on": [
"step_05_net_after_payout"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"document_evidence",
"value_flow_pivot",
"followup_reuse",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)документ|счет|счёт|накладн|акт"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк",
"(?i)2020"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)сколько получили|сколько заплатили|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_07_movements_after_documents",
"title": "Movement evidence follow-up keeps the grounded counterparty after the document answer",
"question": "а по движениям?",
"depends_on": [
"step_06_documents_after_net"
],
"allowed_reply_types": [
"factual",
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"entity_resolution",
"movement_evidence",
"document_pivot",
"followup_reuse",
"planner_catalog_alignment"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)движени|операц|платеж|платёж|списан|поступ"
],
"required_answer_patterns_any": [
"(?i)группа\\s+свк",
"(?i)свк",
"(?i)2020"
],
"forbidden_answer_patterns": [
"(?i)не найден контрагент|уточните, какого контрагента|по какому контрагенту",
"(?i)сколько получили|сколько заплатили|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_08_broad_business_evaluation_explicit_org",
"title": "Broad business evaluation stays grounded and bounded after counterparty drilldowns",
"question": "Как ты оценишь деятельность компании {{bindings.main_organization}} по данным 1С?",
"depends_on": [
"step_07_movements_after_documents"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"broad_business_evaluation",
"grounded_summary",
"context_switch_from_counterparty_to_company"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_all": [
"(?i)альтернатива",
"(?i)ограниченн|проверенн|не\\s+аудит|не\\s+аудитор",
"(?i)1с|подтвержд",
"(?i)денежн|долг|ндс|контрагент|операц"
],
"forbidden_answer_patterns": [
"(?i)^\\s*коротко:",
"(?i)чистая\\s+прибыль\\s+подтверждена|аудиторское\\s+заключение",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "critical"
},
{
"step_id": "step_09_off_domain_living_chat_not_hijacked",
"title": "Off-domain living chat is not hijacked by stale business context",
"question": "а чем капибара отличается от утки?",
"depends_on": [
"step_08_broad_business_evaluation_explicit_org"
],
"allowed_reply_types": [
"factual_with_explanation",
"partial_coverage"
],
"semantic_tags": [
"off_domain_living_chat",
"stale_replay_forbidden",
"context_boundary"
],
"required_answer_shape": "direct_answer_first",
"required_answer_patterns_any": [
"(?i)капибар.*утк|утк.*капибар",
"(?i)млекопита|птиц|грызун"
],
"forbidden_answer_patterns": [
"(?i)альтернатива|свк|контрагент|1с|поступлен|платеж|платёж|нетто",
"(?i)route_candidate|primitive|planner_|catalog_|snapshot_items|answer_object|capability_id|mcp_discovery"
],
"criticality": "warning"
}
]
}

8
docs/orchestration/detector_registry.json
View File

@ -218,7 +218,9 @@
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["steps/<step_id>/output.md"],
"check": {
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
"artifact_path_include_patterns": [
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_05_net_after_payout)"
],
"required_patterns_any": [
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
]
@ -231,7 +233,9 @@
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["steps/<step_id>/output.md"],
"check": {
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
"artifact_path_include_patterns": [
"(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money|step_03_incoming_by_resolved_entity|step_04_payout_switch_by_resolved_entity|step_05_net_after_payout)"
],
"forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"]
}
},

99
scripts/agent_detector_runner.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import argparse
import json
import os
import re
from datetime import datetime, timezone
from pathlib import Path
@ -18,8 +19,35 @@ DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS = [
]
def path_for_io(path: Path) -> str:
if os.name != "nt":
return str(path)
raw_value = str(path)
if raw_value.startswith("\\\\?\\"):
return raw_value
absolute_path = path if path.is_absolute() else Path.cwd() / path
absolute_value = str(absolute_path.absolute())
if absolute_value.startswith("\\\\?\\"):
return absolute_value
if absolute_value.startswith("\\\\"):
return "\\\\?\\UNC\\" + absolute_value.lstrip("\\")
return "\\\\?\\" + absolute_value
def path_exists(path: Path) -> bool:
try:
return os.path.exists(path_for_io(path))
except OSError:
return False
def read_text(path: Path) -> str:
with open(path_for_io(path), "r", encoding="utf-8") as handle:
return handle.read()
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
return json.loads(read_text(path))
def read_json_object(path: Path) -> dict[str, Any]:
@ -31,8 +59,9 @@ def read_json_object(path: Path) -> dict[str, Any]:
def write_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
os.makedirs(path_for_io(path.parent), exist_ok=True)
with open(path_for_io(path), "w", encoding="utf-8") as handle:
handle.write(json.dumps(payload, ensure_ascii=False, indent=2) + "\n")
def utc_now() -> str:
@ -164,7 +193,7 @@ def select_detectors(
def read_text_or_empty(path: Path) -> str:
try:
return path.read_text(encoding="utf-8")
return read_text(path)
except OSError:
return ""
@ -207,7 +236,7 @@ def first_pattern_search(patterns: list[re.Pattern[str]], text: str) -> re.Match
def assistant_text_from_turn_path(path: Path | None) -> str:
if path is None or not path.exists():
if path is None or not path_exists(path):
return ""
payload = read_json_object(path)
assistant_message = payload.get("assistant_message") if isinstance(payload.get("assistant_message"), dict) else {}
@ -264,14 +293,21 @@ def output_turn_path(output_path: Path) -> Path | None:
name = output_path.name
if name == "output.md":
candidate = output_path.with_name("turn.json")
return candidate if candidate.exists() else None
return candidate if path_exists(candidate) else None
if name.endswith("_output.md"):
prefix = name[: -len("_output.md")]
candidate = output_path.with_name(f"{prefix}_turn.json")
return candidate if candidate.exists() else None
return candidate if path_exists(candidate) else None
return None
def output_step_state_path(output_path: Path) -> Path | None:
if output_path.name != "output.md":
return None
candidate = output_path.with_name("step_state.json")
return candidate if path_exists(candidate) else None
def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
outputs: list[dict[str, Any]] = []
seen: set[Path] = set()
@ -292,6 +328,7 @@ def collect_output_artifacts(artifact_dir: Path) -> list[dict[str, Any]]:
"artifact_path": str(path.relative_to(artifact_dir)),
"text": read_text_or_empty(path),
"turn_path": turn_path,
"step_state_path": output_step_state_path(path),
}
)
return outputs
@ -497,6 +534,52 @@ def evaluate_limited_next_action(
return build_result(detector_name, detector, status, message, evidence=failures)
def evaluate_answer_text_shape(
detector_name: str,
detector: dict[str, Any],
outputs: list[dict[str, Any]],
) -> dict[str, Any]:
if not outputs:
return build_result(detector_name, detector, "skipped", "no output.md-style artifacts matched detector scope")
failures: list[dict[str, Any]] = []
reviewed: list[dict[str, Any]] = []
unknown: list[dict[str, Any]] = []
for output in outputs:
step_state_path = output.get("step_state_path")
step_state = read_json_object(step_state_path) if isinstance(step_state_path, Path) else {}
review = step_state.get("business_first_review") if isinstance(step_state.get("business_first_review"), dict) else {}
direct_answer_first_ok = review.get("direct_answer_first_ok")
if direct_answer_first_ok is True:
reviewed.append({"path": output["repo_path"], "direct_answer_first_ok": True})
elif direct_answer_first_ok is False:
failures.append({"path": output["repo_path"], "direct_answer_first_ok": False})
else:
unknown.append({"path": output["repo_path"], "reason": "business_first_review_missing"})
if failures:
return build_result(
detector_name,
detector,
"fail",
"first-line direct answer check failed",
evidence=[*failures, *unknown],
)
if unknown:
return build_result(
detector_name,
detector,
"review",
"direct-answer shape requires business review",
evidence=[*reviewed, *unknown],
)
return build_result(
detector_name,
detector,
"pass",
"business-first step reviews confirm direct-answer-first shape",
evidence=reviewed,
)
def evaluate_trace_guard(
detector_name: str,
detector: dict[str, Any],
@ -686,7 +769,7 @@ def evaluate_detector(
):
return evaluate_composite(detector_name, detector, results_by_name)
if kind == "answer_text_shape":
return evaluate_manual_review(detector_name, detector, "direct-answer shape requires business review")
return evaluate_answer_text_shape(detector_name, detector, scoped_outputs)
return build_result(detector_name, detector, "skipped", f"detector kind is not executable yet: {kind}")

48
scripts/domain_case_loop.py
View File

@ -317,6 +317,7 @@ DEFAULT_INVARIANT_SEVERITY: dict[str, str] = {
"forbidden_recipe_selected": "P0",
"focus_object_missing": "P0",
"wrong_date_scope_state": "P0",
"out_of_window_date_in_answer": "P0",
"direct_answer_missing": "P0",
"top_level_noise_present": "P0",
"business_direct_answer_missing": "P0",
@ -930,6 +931,40 @@ def first_non_empty_lines(text: str, limit: int = 3) -> list[str]:
return output
def expected_single_year_from_date_scope(date_scope: Any) -> str | None:
if not isinstance(date_scope, dict):
return None
if str(date_scope.get("scope") or "").strip().lower() == "all_time":
return None
period_from = normalize_iso_date(date_scope.get("period_from"))
period_to = normalize_iso_date(date_scope.get("period_to"))
if not period_from or not period_to:
return None
from_year = period_from[:4]
to_year = period_to[:4]
if from_year != to_year:
return None
return from_year if re.fullmatch(r"(?:19|20)\d{2}", from_year) else None
def dated_evidence_years_from_answer(text: Any) -> list[str]:
source = repair_text_mojibake(str(text or ""))
years: list[str] = []
for match in re.finditer(r"\b((?:19|20)\d{2})-\d{2}-\d{2}(?:T|\b)", source):
years.append(match.group(1))
for match in re.finditer(r"\b\d{1,2}\.\d{1,2}\.((?:19|20)\d{2})\b", source):
years.append(match.group(1))
return list(dict.fromkeys(years))
def answer_has_out_of_window_dates_for_scope(text: Any, date_scope: Any) -> bool:
expected_year = expected_single_year_from_date_scope(date_scope)
if not expected_year:
return False
years = dated_evidence_years_from_answer(text)
return any(year != expected_year for year in years)
def build_node_contract_index(raw_contract: dict[str, Any]) -> dict[str, dict[str, Any]]:
scenario_tree = raw_contract.get("scenario_tree")
if not isinstance(scenario_tree, dict):
@ -2867,7 +2902,12 @@ def is_validated_clean_meta_chat_answer(
return False
semantic_tags = set(normalize_string_list(state.get("semantic_tags")))
allowed_tags = {
"human_answer",
"meta_smalltalk",
"mcp_discovery_gate_sanity",
"off_domain_living_chat",
"stale_replay_forbidden",
"context_boundary",
"company_selected",
"organization_authority",
"meta_capability",
@ -3112,6 +3152,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
if current_date_scope and current_date_scope != required_filters["as_of_date"]:
violated_invariants.append("wrong_date_scope_state")
if answer_has_out_of_window_dates_for_scope(assistant_text, date_scope):
violated_invariants.append("out_of_window_date_in_answer")
warnings.append("out_of_window_date_in_answer")
if should_require_direct_answer(state):
if not actual_direct_answer or is_top_level_noise_line(actual_direct_answer):
violated_invariants.append("direct_answer_missing")
@ -3193,8 +3237,10 @@ def validate_step_contract(step_state: dict[str, Any]) -> dict[str, Any]:
state["clarification_answer_validated"] = clarification_validated
state["missing_axis_clarification_validated"] = missing_axis_clarification_validated
state["clean_meta_chat_answer_validated"] = clean_meta_chat_validated
effective_execution_status = "exact" if clean_meta_chat_validated else execution_status
state["execution_status"] = effective_execution_status
state["acceptance_status"] = acceptance_status_from_execution(
execution_status,
effective_execution_status,
hard_fail,
(
bounded_validated

177
scripts/test_agent_detector_runner.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import json
import os
import sys
import tempfile
import unittest
@ -287,6 +288,182 @@ class AgentDetectorRunnerTests(unittest.TestCase):
self.assertEqual(len(evidence_paths), 1)
self.assertIn("step_01_margin_root", evidence_paths[0])
def test_counterparty_value_flow_required_surface_scopes_to_current_net_step(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
write_text(
artifact_dir
/ "scenarios"
/ "mixed_planner_counterparty_evidence_and_living_guard"
/ "steps"
/ "step_03_incoming_by_resolved_entity"
/ "output.md",
"Входящие денежные поступления по контрагенту Группа СВК за 2020: 12 093 465 руб.",
)
write_text(
artifact_dir
/ "scenarios"
/ "mixed_planner_counterparty_evidence_and_living_guard"
/ "steps"
/ "step_05_net_after_payout"
/ "output.md",
"По контрагенту Группа СВК за период 2020 получили 12 093 465 руб., заплатили 0 руб.; расчетное нетто в нашу сторону: 12 093 465 руб.",
)
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"counterparty_value_flow_required_surface": {
"kind": "answer_text_required_any",
"automation_level": "automatic",
"description": "Net value-flow answer must surface all money directions.",
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
"inputs": ["output.md"],
"check": {
"artifact_path_include_patterns": ["(?i)step_05_net_after_payout"],
"required_patterns_any": [
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
],
},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["counterparty_value_flow_required_surface"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
evidence_paths = [item["path"] for item in results["results"][0]["evidence"]]
self.assertEqual(len(evidence_paths), 1)
self.assertIn("step_05_net_after_payout", evidence_paths[0])
def test_answer_text_shape_uses_business_first_step_review(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
write_text(step_dir / "output.md", "Direct business answer first.")
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": True}})
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"first_line_not_direct_answer": {
"kind": "answer_text_shape",
"automation_level": "semi_automatic",
"description": "First line should be direct.",
"issue_codes": ["business_direct_answer_missing"],
"inputs": ["output.md"],
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["first_line_not_direct_answer"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
def test_answer_text_shape_fails_when_business_first_review_rejects_first_line(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
step_dir = artifact_dir / "scenarios" / "mixed" / "steps" / "step_01"
write_text(step_dir / "output.md", "Let me inspect the route first.")
write_json(step_dir / "step_state.json", {"business_first_review": {"direct_answer_first_ok": False}})
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"first_line_not_direct_answer": {
"kind": "answer_text_shape",
"automation_level": "semi_automatic",
"description": "First line should be direct.",
"issue_codes": ["business_direct_answer_missing"],
"inputs": ["output.md"],
"check": {"first_line_should_be": "business_answer_or_honest_boundary"},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["first_line_not_direct_answer"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "fail")
self.assertEqual(results["results"][0]["status"], "fail")
def test_step_state_lookup_handles_long_artifact_paths(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
step_dir = (
root
/ "run"
/ "scenarios"
/ ("mixed_planner_counterparty_evidence_and_living_guard_" + "s" * 48)
/ "steps"
/ ("step_04_payout_switch_by_resolved_entity_" + "x" * 80)
)
output_path = step_dir / "output.md"
step_state_path = step_dir / "step_state.json"
try:
os.makedirs(runner.path_for_io(step_dir), exist_ok=True)
with open(runner.path_for_io(output_path), "w", encoding="utf-8") as handle:
handle.write("Direct business answer first.")
runner.write_json(step_state_path, {"business_first_review": {"direct_answer_first_ok": True}})
self.assertGreater(len(str(step_state_path.absolute())), 260)
self.assertEqual(runner.output_step_state_path(output_path), step_state_path)
self.assertEqual(
runner.read_json_object(step_state_path).get("business_first_review"),
{"direct_answer_first_ok": True},
)
finally:
for path in (output_path, step_state_path):
try:
os.remove(runner.path_for_io(path))
except OSError:
pass
current = step_dir
while current != root:
try:
os.rmdir(runner.path_for_io(current))
except OSError:
pass
current = current.parent
def test_composite_detector_fails_after_child_detector_fails(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)

101
scripts/test_domain_case_loop_step_state.py
View File

@ -153,6 +153,81 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
self.assertEqual(dcl.derive_scenario_execution_status(step_outputs), "partial")
self.assertEqual(dcl.derive_scenario_status(step_outputs), "accepted")
def test_clean_meta_smalltalk_is_exact_without_1c_capability(self) -> None:
step_state = dcl.build_scenario_step_state(
scenario_id="meta_chat_demo",
domain="agentic_loop",
step={
"step_id": "step_01",
"title": "Human smalltalk",
"depends_on": [],
"question_template": "привет, ты на связи?",
"semantic_tags": ["human_answer", "meta_smalltalk", "mcp_discovery_gate_sanity"],
"required_answer_shape": "direct_answer_first",
},
step_index=1,
question_resolved="привет, ты на связи?",
analysis_context={},
turn_artifact={
"assistant_message": {
"reply_type": "factual_with_explanation",
"text": "Привет! Да, я на связи. Готов помочь с анализом данных из 1С в режиме чтения.",
"message_id": "msg-1",
"trace_id": "trace-1",
},
"technical_debug_payload": {
"detected_mode": "chat",
"fallback_type": "none",
"living_chat_response_source": "llm_chat",
},
"session_summary": {},
},
entries=[],
)
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
self.assertEqual(step_state["execution_status"], "exact")
self.assertEqual(step_state["acceptance_status"], "validated")
def test_off_domain_living_chat_is_exact_without_1c_capability(self) -> None:
step_state = dcl.build_scenario_step_state(
scenario_id="off_domain_demo",
domain="agentic_loop",
step={
"step_id": "step_09",
"title": "Off-domain living chat",
"depends_on": [],
"question_template": "а чем капибара отличается от утки?",
"semantic_tags": ["off_domain_living_chat", "stale_replay_forbidden", "context_boundary"],
"required_answer_shape": "direct_answer_first",
},
step_index=9,
question_resolved="а чем капибара отличается от утки?",
analysis_context={},
turn_artifact={
"assistant_message": {
"reply_type": "factual_with_explanation",
"text": (
"Капибара и утка отличаются принципиально: капибара - млекопитающее-грызун, "
"а утка - птица. Поэтому у них разные тело, среда обитания и способ передвижения."
),
"message_id": "msg-2",
"trace_id": "trace-2",
},
"technical_debug_payload": {
"detected_mode": "chat",
"fallback_type": "none",
"living_chat_response_source": "llm_chat",
},
"session_summary": {},
},
entries=[],
)
self.assertTrue(step_state["clean_meta_chat_answer_validated"])
self.assertEqual(step_state["execution_status"], "exact")
self.assertEqual(step_state["acceptance_status"], "validated")
def test_today_scope_required_filter_and_direct_patterns_are_enforced(self) -> None:
self.assertTrue(dcl.question_resets_temporal_scope("мы должны комуто денег на сегодня?"))
@ -610,6 +685,32 @@ class DomainCaseLoopStepStateTests(unittest.TestCase):
self.assertIsNone(step_state["date_scope"]["as_of_date"])
self.assertEqual(step_state["date_scope"]["source"], "question_temporal_scope_reset")
def test_out_of_window_document_dates_reject_validated_step(self) -> None:
validated = dcl.validate_step_contract(
{
"execution_status": "exact",
"required_answer_shape": "direct_answer_first",
"reply_type": "factual",
"actual_direct_answer": "Контрагент: Группа СВК. Найдено документов: 19.",
"assistant_text": (
"Контрагент: Группа СВК. Найдено документов: 19.\n"
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07"
),
"top_non_empty_lines": [
"Контрагент: Группа СВК. Найдено документов: 19.",
"1. 2021-11-10T12:00:07Z | Поступление на расчетный счет 00000000013 от 10.11.2021 12:00:07",
],
"date_scope": {
"period_from": "2020-01-01",
"period_to": "2020-12-31",
},
}
)
self.assertEqual(validated["status"], "rejected")
self.assertIn("out_of_window_date_in_answer", validated["violated_invariants"])
self.assertIn("out_of_window_date_in_answer", validated["warnings"])
def test_open_items_exact_negative_answer_validates_without_rows(self) -> None:
step_state = dcl.build_scenario_step_state(
scenario_id="open_items_negative_demo",