Укрепить агентный луп для проверки контуров 1С
This commit is contained in:
parent
9e2bc8cb6c
commit
77c88a018a
File diff suppressed because it is too large
Load Diff
|
|
@ -208,6 +208,41 @@
|
|||
"forbidden_patterns": ["(?im)^\\s*(Что проверили|Опорные документы):"]
|
||||
}
|
||||
},
|
||||
"counterparty_value_flow_required_surface": {
|
||||
"kind": "answer_text_required_any",
|
||||
"automation_level": "automatic",
|
||||
"description": "Counterparty value-flow answers must surface counterparty scope, incoming amount, outgoing amount, and net/saldo wording.",
|
||||
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||
"inputs": ["steps/<step_id>/output.md"],
|
||||
"check": {
|
||||
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
|
||||
"required_patterns_any": [
|
||||
"(?is)(?=.*(СВК|Группа\\s+СВК))(?=.*(входящ|получил|получено|получили))(?=.*(исходящ|заплатил|заплачено|заплатили|ушло))(?=.*(нетто|сальдо|разниц|чистый\\s+денежный))"
|
||||
]
|
||||
}
|
||||
},
|
||||
"counterparty_value_flow_profit_accounts_forbidden": {
|
||||
"kind": "answer_text_regex_forbidden",
|
||||
"automation_level": "automatic",
|
||||
"description": "Counterparty received/paid/net-flow answers must not use company profit or 90/91/99 accounts as the direct answer.",
|
||||
"issue_codes": ["counterparty_value_flow_misrouted_to_company_profit"],
|
||||
"inputs": ["steps/<step_id>/output.md"],
|
||||
"check": {
|
||||
"artifact_path_include_patterns": ["(?i)(s01_svk_money_documents|s01_select_svk_counterparty_money)"],
|
||||
"forbidden_patterns": ["(?i)(сч[её]т\\s*(90|91|99)|90[\\./](01|02|09)|91[\\./]|99\\b|company[- ]level\\s+profit)"]
|
||||
}
|
||||
},
|
||||
"bank_counterparty_boundary_required": {
|
||||
"kind": "answer_text_required_any",
|
||||
"automation_level": "semi_automatic",
|
||||
"description": "Money-leader answers that explicitly mention bank role must include a boundary instead of presenting a bank as an ordinary customer/supplier.",
|
||||
"issue_codes": ["bank_counterparty_misclassified_as_business_partner"],
|
||||
"inputs": ["steps/<step_id>/output.md"],
|
||||
"check": {
|
||||
"artifact_path_include_patterns": ["(?i)s02_colloquial_money_leaders"],
|
||||
"required_patterns_any": ["(?is)(банк|банковск|финансов|не\\s+обычн|роль|назначени[ея]|плат[её]ж)"]
|
||||
}
|
||||
},
|
||||
"route_candidate_needs_enablement": {
|
||||
"kind": "stage_review_signal",
|
||||
"automation_level": "semi_automatic",
|
||||
|
|
|
|||
|
|
@ -411,7 +411,11 @@
|
|||
"severity": "P0",
|
||||
"business_meaning": "Контрагентский вопрос про получили/заплатили/нетто отвечен company-level прибылью или 90/91/99 вместо контрагентского денежного потока.",
|
||||
"root_layers": ["followup_action_resolution_gap", "answer_shape_mismatch", "domain_scope_leak"],
|
||||
"detectors": ["counterparty_value_flow_profit_mismatch"],
|
||||
"detectors": [
|
||||
"counterparty_value_flow_required_surface",
|
||||
"counterparty_value_flow_profit_accounts_forbidden",
|
||||
"counterparty_value_flow_profit_mismatch"
|
||||
],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryRuntimeBridge.ts",
|
||||
|
|
@ -489,7 +493,10 @@
|
|||
"severity": "P1",
|
||||
"business_meaning": "Банк/финансовая организация ошибочно подана как обычный клиент/поставщик без роли, назначения платежа и договорного контекста.",
|
||||
"root_layers": ["business_semantic_role_gap", "domain_scope"],
|
||||
"detectors": ["bank_counterparty_role_boundary_missing"],
|
||||
"detectors": [
|
||||
"bank_counterparty_boundary_required",
|
||||
"bank_counterparty_role_boundary_missing"
|
||||
],
|
||||
"allowed_patch_targets": [
|
||||
"llm_normalizer/backend/src/services/address_runtime/composeStage.ts",
|
||||
"llm_normalizer/backend/src/services/assistantMcpDiscoveryResponseCandidate.ts",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ DETECTOR_REGISTRY_PATH = REPO_ROOT / "docs" / "orchestration" / "detector_regist
|
|||
ISSUE_CATALOG_PATH = REPO_ROOT / "docs" / "orchestration" / "issue_catalog.json"
|
||||
DEFAULT_GLOBAL_DETECTORS = ["missing_effective_runtime_json"]
|
||||
DETECTOR_RESULTS_SCHEMA_VERSION = "agent_detector_results_v1"
|
||||
DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS = [
|
||||
r"(?i)(что\s+проверить\s+дальше|для\s+ответа.{0,120}нужно|нужно\s+отдельно\s+(?:считать|посчитать|проверить)|проверить\s+дальше)"
|
||||
]
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
|
|
@ -166,6 +169,43 @@ def read_text_or_empty(path: Path) -> str:
|
|||
return ""
|
||||
|
||||
|
||||
def decode_win1251_utf8_mojibake(value: str) -> str | None:
|
||||
try:
|
||||
return value.encode("cp1251").decode("utf-8")
|
||||
except UnicodeError:
|
||||
return None
|
||||
|
||||
|
||||
def decode_latin1_utf8_mojibake(value: str) -> str | None:
|
||||
try:
|
||||
return value.encode("latin1").decode("utf-8")
|
||||
except UnicodeError:
|
||||
return None
|
||||
|
||||
|
||||
def text_match_variants(value: str) -> list[str]:
|
||||
variants: list[str] = []
|
||||
queue = [str(value or "")]
|
||||
while queue:
|
||||
current = queue.pop(0)
|
||||
if current in variants:
|
||||
continue
|
||||
variants.append(current)
|
||||
for decoded in (decode_win1251_utf8_mojibake(current), decode_latin1_utf8_mojibake(current)):
|
||||
if decoded and decoded not in variants and decoded not in queue:
|
||||
queue.append(decoded)
|
||||
return variants
|
||||
|
||||
|
||||
def first_pattern_search(patterns: list[re.Pattern[str]], text: str) -> re.Match[str] | None:
|
||||
for variant in text_match_variants(text):
|
||||
for pattern in patterns:
|
||||
match = pattern.search(variant)
|
||||
if match:
|
||||
return match
|
||||
return None
|
||||
|
||||
|
||||
def assistant_text_from_turn_path(path: Path | None) -> str:
|
||||
if path is None or not path.exists():
|
||||
return ""
|
||||
|
|
@ -384,7 +424,7 @@ def evaluate_forbidden_regex(
|
|||
lines = [line for line in text.splitlines() if line.strip()]
|
||||
text = "\n".join(lines[:prefix_line_count])
|
||||
for pattern in patterns:
|
||||
match = pattern.search(text)
|
||||
match = first_pattern_search([pattern], text)
|
||||
if match:
|
||||
evidence.append(
|
||||
{
|
||||
|
|
@ -411,7 +451,7 @@ def evaluate_required_any(
|
|||
matched: list[dict[str, Any]] = []
|
||||
for output in outputs:
|
||||
text = current_answer_text_for_output(output)
|
||||
match = next((pattern.search(text) for pattern in patterns if pattern.search(text)), None)
|
||||
match = first_pattern_search(patterns, text)
|
||||
if match:
|
||||
matched.append({"path": output["repo_path"], "match": match.group(0)[:240]})
|
||||
else:
|
||||
|
|
@ -428,18 +468,24 @@ def evaluate_limited_next_action(
|
|||
) -> dict[str, Any]:
|
||||
check = detector.get("check") if isinstance(detector.get("check"), dict) else {}
|
||||
limited_patterns = compile_patterns(normalize_string_list(check.get("limited_patterns")))
|
||||
next_action_patterns = compile_patterns(normalize_string_list(check.get("required_next_action_patterns_any")))
|
||||
next_action_pattern_sources = [
|
||||
*normalize_string_list(check.get("required_next_action_patterns_any")),
|
||||
*normalize_string_list(check.get("required_next_action_patterns_extra_any")),
|
||||
]
|
||||
if detector_name == "limited_answer_without_next_action":
|
||||
next_action_pattern_sources.extend(DEFAULT_LIMITED_NEXT_ACTION_EXTRA_PATTERNS)
|
||||
next_action_patterns = compile_patterns(next_action_pattern_sources)
|
||||
if not outputs:
|
||||
return build_result(detector_name, detector, "skipped", "no output.md-style artifacts matched detector scope")
|
||||
failures: list[dict[str, Any]] = []
|
||||
limited_outputs = 0
|
||||
for output in outputs:
|
||||
text = current_answer_text_for_output(output)
|
||||
limited_match = next((pattern.search(text) for pattern in limited_patterns if pattern.search(text)), None)
|
||||
limited_match = first_pattern_search(limited_patterns, text)
|
||||
if not limited_match:
|
||||
continue
|
||||
limited_outputs += 1
|
||||
if not any(pattern.search(text) for pattern in next_action_patterns):
|
||||
if not first_pattern_search(next_action_patterns, text):
|
||||
failures.append({"path": output["repo_path"], "limited_match": limited_match.group(0)[:240]})
|
||||
status = "fail" if failures else "pass"
|
||||
if failures:
|
||||
|
|
|
|||
|
|
@ -6225,7 +6225,22 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit
|
|||
return names[:limit]
|
||||
|
||||
status = str(summary.get("status") or "skipped")
|
||||
declared_detector_count = len(normalize_string_list(detector_results.get("declared_detectors_under_test")))
|
||||
declared_detectors = normalize_string_list(detector_results.get("declared_detectors_under_test"))
|
||||
declared_detector_count = len(declared_detectors)
|
||||
|
||||
def _declared_with_status(status: str) -> list[str]:
|
||||
names: list[str] = []
|
||||
declared = set(declared_detectors)
|
||||
if not declared:
|
||||
return names
|
||||
for item in results:
|
||||
if not isinstance(item, dict) or str(item.get("status") or "") != status:
|
||||
continue
|
||||
detector_name = str(item.get("detector") or "").strip()
|
||||
if detector_name in declared and detector_name not in names:
|
||||
names.append(detector_name)
|
||||
return names[:limit]
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"detector_count": int(summary.get("detector_count") or len(results)),
|
||||
|
|
@ -6237,6 +6252,7 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit
|
|||
"failed_detectors": _detectors_with_status("fail"),
|
||||
"review_detectors": _detectors_with_status("review"),
|
||||
"skipped_detectors": _detectors_with_status("skipped"),
|
||||
"declared_skipped_detectors": _declared_with_status("skipped"),
|
||||
"signal_ok_for_auto_coder": status in {"fail", "review"},
|
||||
}
|
||||
|
||||
|
|
@ -6247,6 +6263,9 @@ def evaluate_detector_results_gate(detector_results_summary: dict[str, Any]) ->
|
|||
detector_count = int(detector_results_summary.get("detector_count") or 0)
|
||||
if declared_detector_count > 0 and detector_count == 0:
|
||||
return False, "declared_detector_results_missing"
|
||||
declared_skipped = normalize_string_list(detector_results_summary.get("declared_skipped_detectors"))
|
||||
if declared_skipped:
|
||||
return False, f"declared_detector_results_skipped:{','.join(declared_skipped)}"
|
||||
if declared_detector_count > 0 and status == "skipped":
|
||||
return False, "declared_detector_results_skipped"
|
||||
if status == "fail":
|
||||
|
|
|
|||
|
|
@ -22,6 +22,10 @@ def write_text(path: Path, text: str) -> None:
|
|||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def to_win1251_mojibake(value: str) -> str:
|
||||
return value.encode("utf-8").decode("cp1251")
|
||||
|
||||
|
||||
class AgentDetectorRunnerTests(unittest.TestCase):
|
||||
def test_default_runner_fails_missing_effective_runtime(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
|
|
@ -102,6 +106,59 @@ class AgentDetectorRunnerTests(unittest.TestCase):
|
|||
self.assertEqual(results["summary"]["status"], "pass")
|
||||
self.assertEqual(results["results"][0]["status"], "pass")
|
||||
|
||||
def test_limited_next_action_accepts_mojibake_what_to_check_next(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
artifact_dir = root / "run"
|
||||
answer = (
|
||||
"Это не подтвержденная чистая прибыль.\n"
|
||||
"Что проверить дальше: чистую прибыль через счета 90/91/99 и закрытие периода."
|
||||
)
|
||||
write_text(artifact_dir / "steps" / "s01" / "output.md", to_win1251_mojibake(answer))
|
||||
registry_path = root / "detector_registry.json"
|
||||
issue_catalog_path = root / "issue_catalog.json"
|
||||
write_json(
|
||||
registry_path,
|
||||
{
|
||||
"schema_version": "agent_detector_registry_v1",
|
||||
"detectors": {
|
||||
"limited_answer_without_next_action": {
|
||||
"kind": "answer_text_required_when_limited",
|
||||
"automation_level": "semi_automatic",
|
||||
"description": "Limited answers need a next action.",
|
||||
"issue_codes": ["business_next_step_missing"],
|
||||
"inputs": ["output.md"],
|
||||
"check": {
|
||||
"limited_patterns": ["(?i)не подтвержден"],
|
||||
"required_next_action_patterns_any": ["(?i)(можно|проверь)"],
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
write_json(
|
||||
issue_catalog_path,
|
||||
{
|
||||
"schema_version": "agent_issue_catalog_v1",
|
||||
"issues": {
|
||||
"business_next_step_missing": {
|
||||
"detectors": ["limited_answer_without_next_action"],
|
||||
}
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
results = runner.build_detector_results(
|
||||
artifact_dir,
|
||||
issue_codes=["business_next_step_missing"],
|
||||
registry_path=registry_path,
|
||||
issue_catalog_path=issue_catalog_path,
|
||||
include_default_global=False,
|
||||
)
|
||||
|
||||
self.assertEqual(results["summary"]["status"], "pass")
|
||||
self.assertEqual(results["results"][0]["message"], "limited answer includes a next action")
|
||||
|
||||
def test_candidate_forbidden_regex_is_limited_to_evidence_path(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
|
|
|
|||
|
|
@ -404,6 +404,24 @@ class DomainCaseLoopLeadHandoffTests(unittest.TestCase):
|
|||
self.assertEqual(summary["declared_detector_count"], 1)
|
||||
self.assertEqual(reason, "declared_detector_results_missing")
|
||||
|
||||
def test_detector_gate_blocks_declared_detector_skipped_inside_mixed_pass_run(self) -> None:
|
||||
summary = dcl.summarize_detector_results(
|
||||
{
|
||||
"summary": {"status": "pass", "detector_count": 2, "pass": 1, "fail": 0, "review": 0, "skipped": 1},
|
||||
"results": [
|
||||
{"detector": "runtime_tokens_in_user_answer", "status": "pass"},
|
||||
{"detector": "counterparty_value_flow_profit_mismatch", "status": "skipped"},
|
||||
],
|
||||
"declared_detectors_under_test": ["counterparty_value_flow_profit_mismatch"],
|
||||
}
|
||||
)
|
||||
|
||||
ok, reason = dcl.evaluate_detector_results_gate(summary)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(summary["declared_skipped_detectors"], ["counterparty_value_flow_profit_mismatch"])
|
||||
self.assertEqual(reason, "declared_detector_results_skipped:counterparty_value_flow_profit_mismatch")
|
||||
|
||||
def test_auto_coder_gate_allows_when_detector_results_confirm_failure(self) -> None:
|
||||
repair_targets = {
|
||||
"targets": [
|
||||
|
|
|
|||
Loading…
Reference in New Issue