From 9e2bc8cb6cf52cf2bba6905358db7b5037f392cc Mon Sep 17 00:00:00 2001 From: dctouch Date: Mon, 15 Jun 2026 17:18:58 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A3=D1=81=D0=B8=D0=BB=D0=B8=D1=82=D1=8C=20?= =?UTF-8?q?=D0=B0=D0=B3=D0=B5=D0=BD=D1=82=D0=BD=D1=8B=D0=B9=20detector=20g?= =?UTF-8?q?ate=20=D0=B4=D0=BB=D1=8F=20declared=20detectors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/orchestration/detector_registry.json | 6 ++ scripts/agent_detector_runner.py | 20 +++- scripts/domain_case_loop.py | 98 ++++++++++++++++--- scripts/stage_agent_loop.py | 14 +++ scripts/test_agent_detector_runner.py | 54 ++++++++++ scripts/test_domain_case_loop_lead_handoff.py | 51 ++++++++++ scripts/test_stage_agent_loop.py | 7 ++ 7 files changed, 234 insertions(+), 16 deletions(-) diff --git a/docs/orchestration/detector_registry.json b/docs/orchestration/detector_registry.json index f3043b5..2a2bcc2 100644 --- a/docs/orchestration/detector_registry.json +++ b/docs/orchestration/detector_registry.json @@ -65,6 +65,12 @@ "check": { "required_patterns_any": [ "(?i)(выруч|себестоим|валов|марж|не хватает|не могу подтвердить|не подтвержден|unknown)" + ], + "artifact_path_exclude_patterns": [ + "(?i)selected_item", + "(?i)supplier", + "(?i)purchase_date", + "(?i)purchase_document" ] } }, diff --git a/scripts/agent_detector_runner.py b/scripts/agent_detector_runner.py index 55ba2b1..104f478 100644 --- a/scripts/agent_detector_runner.py +++ b/scripts/agent_detector_runner.py @@ -295,6 +295,22 @@ def filter_turns(turns: list[dict[str, Any]], artifact_dir: Path, evidence_paths return [item for item in turns if path_matches_evidence(item["path"], artifact_dir, evidence_paths)] +def filter_artifacts_by_detector_scope(items: list[dict[str, Any]], check: dict[str, Any]) -> list[dict[str, Any]]: + include_patterns = compile_patterns(normalize_string_list(check.get("artifact_path_include_patterns"))) + exclude_patterns = compile_patterns(normalize_string_list(check.get("artifact_path_exclude_patterns"))) + if not include_patterns and not exclude_patterns: + return items + filtered: list[dict[str, Any]] = [] + for item in items: + artifact_path = str(item.get("artifact_path") or item.get("repo_path") or "") + if include_patterns and not any(pattern.search(artifact_path) for pattern in include_patterns): + continue + if exclude_patterns and any(pattern.search(artifact_path) for pattern in exclude_patterns): + continue + filtered.append(item) + return filtered + + def compile_patterns(patterns: list[str]) -> list[re.Pattern[str]]: compiled: list[re.Pattern[str]] = [] for pattern in patterns: @@ -591,8 +607,8 @@ def evaluate_detector( ) -> dict[str, Any]: kind = str(detector.get("kind") or "").strip() check = detector.get("check") if isinstance(detector.get("check"), dict) else {} - scoped_outputs = filter_outputs(outputs, artifact_dir, evidence_paths) - scoped_turns = filter_turns(turns, artifact_dir, evidence_paths) + scoped_outputs = filter_artifacts_by_detector_scope(filter_outputs(outputs, artifact_dir, evidence_paths), check) + scoped_turns = filter_artifacts_by_detector_scope(filter_turns(turns, artifact_dir, evidence_paths), check) if kind == "artifact_presence": return evaluate_artifact_presence(detector_name, detector, artifact_dir) if kind == "answer_text_regex_forbidden": diff --git a/scripts/domain_case_loop.py b/scripts/domain_case_loop.py index 660f4be..473f98f 100644 --- a/scripts/domain_case_loop.py +++ b/scripts/domain_case_loop.py @@ -6107,11 +6107,45 @@ def build_accepted_pack_smoke_detector_candidates() -> list[dict[str, Any]]: ] -def build_detector_candidates(repair_targets: dict[str, Any], catalog: dict[str, Any] | None = None) -> dict[str, Any]: +def build_detector_candidates( + repair_targets: dict[str, Any], + catalog: dict[str, Any] | None = None, + *, + declared_detectors: list[str] | None = None, + declared_issue_codes: list[str] | None = None, +) -> dict[str, Any]: source = catalog if isinstance(catalog, dict) else load_issue_catalog() issues = source.get("issues") if isinstance(source.get("issues"), dict) else {} candidates: list[dict[str, Any]] = [] seen: set[tuple[str, str]] = set() + + def add_candidate( + *, + issue_code: str, + detector: str, + severity: Any = None, + sample_target_id: str | None = None, + evidence_paths: list[str] | None = None, + source_label: str, + ) -> None: + detector_name = str(detector or "").strip() + if not detector_name: + return + key = (str(issue_code or "").strip(), detector_name) + if key in seen: + return + seen.add(key) + candidates.append( + { + "issue_code": key[0], + "detector": detector_name, + "severity": severity, + "sample_target_id": sample_target_id, + "evidence_paths": evidence_paths or [], + "source": source_label, + } + ) + for target in repair_targets.get("targets") if isinstance(repair_targets.get("targets"), list) else []: if not isinstance(target, dict): continue @@ -6122,24 +6156,42 @@ def build_detector_candidates(repair_targets: dict[str, Any], catalog: dict[str, detectors = [f"{issue_code}_detector"] evidence_paths = detector_evidence_paths_for_target(target) for detector in detectors: - key = (issue_code, detector) - if key in seen: - continue - seen.add(key) - candidates.append( - { - "issue_code": issue_code, - "detector": detector, - "severity": target.get("severity"), - "sample_target_id": target.get("target_id"), - "evidence_paths": evidence_paths, - } + add_candidate( + issue_code=issue_code, + detector=detector, + severity=target.get("severity"), + sample_target_id=target.get("target_id"), + evidence_paths=evidence_paths, + source_label="repair_target", ) + declared_codes = normalize_string_list(declared_issue_codes) + detector_to_declared_issue: dict[str, str] = {} + for issue_code in declared_codes: + entry = issues.get(issue_code) if isinstance(issues.get(issue_code), dict) else {} + for detector in normalize_string_list(entry.get("detectors")): + detector_to_declared_issue.setdefault(detector, issue_code) + add_candidate( + issue_code=issue_code, + detector=detector, + severity=entry.get("severity"), + sample_target_id="manifest.issue_codes_under_test", + source_label="manifest.issue_codes_under_test", + ) + for detector in normalize_string_list(declared_detectors): + issue_code = detector_to_declared_issue.get(detector, "") + add_candidate( + issue_code=issue_code, + detector=detector, + sample_target_id="manifest.detectors_under_test", + source_label="manifest.detectors_under_test", + ) if should_add_accepted_pack_smoke_detectors(repair_targets, candidates): candidates.extend(build_accepted_pack_smoke_detector_candidates()) return { "schema_version": "detector_candidates_v1", "candidate_count": len(candidates), + "declared_detector_count": len(normalize_string_list(declared_detectors)), + "declared_issue_code_count": len(declared_codes), "candidates": candidates, } @@ -6149,6 +6201,7 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit return { "status": "not_run", "detector_count": 0, + "declared_detector_count": 0, "pass": 0, "fail": 0, "review": 0, @@ -6172,9 +6225,11 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit return names[:limit] status = str(summary.get("status") or "skipped") + declared_detector_count = len(normalize_string_list(detector_results.get("declared_detectors_under_test"))) return { "status": status, "detector_count": int(summary.get("detector_count") or len(results)), + "declared_detector_count": declared_detector_count, "pass": int(summary.get("pass") or 0), "fail": int(summary.get("fail") or 0), "review": int(summary.get("review") or 0), @@ -6188,6 +6243,12 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit def evaluate_detector_results_gate(detector_results_summary: dict[str, Any]) -> tuple[bool, str]: status = str(detector_results_summary.get("status") or "not_run").strip() + declared_detector_count = int(detector_results_summary.get("declared_detector_count") or 0) + detector_count = int(detector_results_summary.get("detector_count") or 0) + if declared_detector_count > 0 and detector_count == 0: + return False, "declared_detector_results_missing" + if declared_detector_count > 0 and status == "skipped": + return False, "declared_detector_results_skipped" if status == "fail": failed = normalize_string_list(detector_results_summary.get("failed_detectors")) suffix = ",".join(failed) if failed else str(detector_results_summary.get("fail") or 0) @@ -6894,6 +6955,7 @@ def build_loop_final_status(loop_state: dict[str, Any]) -> str: def handle_run_pack_loop(args: argparse.Namespace) -> int: manifest_path = Path(args.manifest).resolve() + pack_manifest = read_json_file(manifest_path) loop_id = str(args.loop_id or slugify_case_id("domain_pack_loop", None)).strip() loop_dir = Path(args.output_root).resolve() / loop_id iterations_dir = loop_dir / "iterations" @@ -7026,13 +7088,21 @@ def handle_run_pack_loop(args: argparse.Namespace) -> int: "source_repair_targets": repo_relative(repair_targets_path), "items": collect_rerun_matrix(repair_targets), } - detector_candidates = build_detector_candidates(repair_targets) + declared_detectors = normalize_string_list(pack_manifest.get("detectors_under_test")) + declared_issue_codes = normalize_string_list(pack_manifest.get("issue_codes_under_test")) + detector_candidates = build_detector_candidates( + repair_targets, + declared_detectors=declared_detectors, + declared_issue_codes=declared_issue_codes, + ) write_json(detector_candidates_path, detector_candidates) detector_results = agent_detector_runner.build_detector_results( pack_dir, detector_candidates_path=detector_candidates_path, include_default_global=False, ) + detector_results["declared_detectors_under_test"] = declared_detectors + detector_results["declared_issue_codes_under_test"] = declared_issue_codes write_json(detector_results_path, detector_results) detector_results_summary = summarize_detector_results(detector_results) detector_results_gate_ok, detector_results_gate_reason = evaluate_detector_results_gate(detector_results_summary) diff --git a/scripts/stage_agent_loop.py b/scripts/stage_agent_loop.py index fddd891..8c48b86 100644 --- a/scripts/stage_agent_loop.py +++ b/scripts/stage_agent_loop.py @@ -1830,6 +1830,16 @@ def build_replay_validation_stage_summary( "global_plan_refs": stage_manifest.get("global_plan_refs") or base.get("global_plan_refs") or [], "target_score": stage_manifest.get("target_score", base.get("target_score")), "acceptance_invariants": stage_manifest.get("acceptance_invariants") or base.get("acceptance_invariants") or [], + "loop_final_status": validation.get("final_status") or base.get("loop_final_status"), + "stop_reason": "accepted_validation_ingested" if accepted else "validation_not_accepted", + "iterations_ran": base.get("iterations_ran") or (1 if validation.get("last_iteration_id") else None), + "last_quality_score": validation.get("quality_score", base.get("last_quality_score")), + "last_analyst_decision": validation.get("review_overall_status") or base.get("last_analyst_decision"), + "last_deterministic_gate_ok": validation.get("deterministic_gate_ok", base.get("last_deterministic_gate_ok")), + "last_deterministic_gate_reason": validation.get( + "deterministic_gate_reason", + base.get("last_deterministic_gate_reason"), + ), "accepted_gate": accepted, "manual_confirmation_required": accepted, "next_action": next_action, @@ -2608,6 +2618,10 @@ def build_stage_status(stage_manifest: dict[str, Any], stage_dir: Path) -> dict[ "summary_exists": bool(summary), "repair_mode": repair_mode, "loop_final_status": summary.get("loop_final_status"), + "last_quality_score": summary.get("last_quality_score"), + "last_analyst_decision": summary.get("last_analyst_decision"), + "last_deterministic_gate_ok": summary.get("last_deterministic_gate_ok"), + "last_deterministic_gate_reason": summary.get("last_deterministic_gate_reason"), "accepted_gate": summary.get("accepted_gate"), "loop_accepted_gate": summary.get("loop_accepted_gate"), "stage_closing_gate": stage_closing_gate or None, diff --git a/scripts/test_agent_detector_runner.py b/scripts/test_agent_detector_runner.py index 44174b2..0165717 100644 --- a/scripts/test_agent_detector_runner.py +++ b/scripts/test_agent_detector_runner.py @@ -166,6 +166,60 @@ class AgentDetectorRunnerTests(unittest.TestCase): self.assertEqual(results["summary"]["status"], "pass") self.assertEqual(results["results"][0]["status"], "pass") + def test_required_any_detector_can_exclude_selected_object_followup_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + artifact_dir = root / "run" + write_text( + artifact_dir / "scenarios" / "margin_root" / "steps" / "step_01" / "output.md", + "Top margin item: revenue 100, cogs 60, gross profit 40, margin 40%.", + ) + write_text( + artifact_dir + / "scenarios" + / "margin_followup" + / "steps" + / "step_03_selected_item_supplier" + / "output.md", + "Supplier for the selected item: Example LLC.", + ) + registry_path = root / "detector_registry.json" + issue_catalog_path = root / "issue_catalog.json" + write_json( + registry_path, + { + "schema_version": "agent_detector_registry_v1", + "detectors": { + "missing_revenue_cogs_margin_fields": { + "kind": "answer_text_required_any", + "automation_level": "semi_automatic", + "description": "Root margin answers need margin fields.", + "issue_codes": ["margin_domain_leak_accounting_route"], + "inputs": ["output.md"], + "check": { + "required_patterns_any": ["(?i)(revenue|cogs|gross profit|margin)"], + "artifact_path_exclude_patterns": ["(?i)selected_item"], + }, + } + }, + }, + ) + write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}}) + + results = runner.build_detector_results( + artifact_dir, + detector_names=["missing_revenue_cogs_margin_fields"], + registry_path=registry_path, + issue_catalog_path=issue_catalog_path, + include_default_global=False, + ) + + self.assertEqual(results["summary"]["status"], "pass") + self.assertEqual(results["results"][0]["status"], "pass") + evidence_paths = [item["path"] for item in results["results"][0]["evidence"]] + self.assertEqual(len(evidence_paths), 1) + self.assertIn("margin_root", evidence_paths[0]) + def test_composite_detector_fails_after_child_detector_fails(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/scripts/test_domain_case_loop_lead_handoff.py b/scripts/test_domain_case_loop_lead_handoff.py index 4369c35..2d1fdb5 100644 --- a/scripts/test_domain_case_loop_lead_handoff.py +++ b/scripts/test_domain_case_loop_lead_handoff.py @@ -353,6 +353,57 @@ class DomainCaseLoopLeadHandoffTests(unittest.TestCase): self.assertEqual(gate["detector_results_summary"]["status"], "pass") self.assertIn("detector_results_no_repair_signal:pass", gate["blocking_reasons"]) + def test_detector_candidates_include_manifest_declared_detectors_without_repair_targets(self) -> None: + repair_targets = { + "target_count": 0, + "acceptance_status": "accepted", + "final_status": "accepted", + "targets": [], + } + catalog = { + "issues": { + "technical_garbage_in_answer": { + "severity": "P0", + "detectors": ["runtime_tokens_in_user_answer", "capability_ids_in_user_answer"], + }, + "margin_domain_leak_accounting_route": { + "severity": "P0", + "detectors": ["forbidden_margin_terms"], + }, + } + } + + candidates = dcl.build_detector_candidates( + repair_targets, + catalog=catalog, + declared_issue_codes=["technical_garbage_in_answer"], + declared_detectors=["forbidden_margin_terms"], + ) + detector_names = [item["detector"] for item in candidates["candidates"]] + source_by_detector = {item["detector"]: item["source"] for item in candidates["candidates"]} + + self.assertEqual(candidates["declared_detector_count"], 1) + self.assertEqual(candidates["declared_issue_code_count"], 1) + self.assertIn("runtime_tokens_in_user_answer", detector_names) + self.assertIn("capability_ids_in_user_answer", detector_names) + self.assertIn("forbidden_margin_terms", detector_names) + self.assertEqual(source_by_detector["forbidden_margin_terms"], "manifest.detectors_under_test") + + def test_detector_gate_blocks_empty_run_when_manifest_declares_detectors(self) -> None: + summary = dcl.summarize_detector_results( + { + "summary": {"status": "skipped", "detector_count": 0, "pass": 0, "fail": 0, "review": 0, "skipped": 0}, + "results": [], + "declared_detectors_under_test": ["forbidden_margin_terms"], + } + ) + + ok, reason = dcl.evaluate_detector_results_gate(summary) + + self.assertFalse(ok) + self.assertEqual(summary["declared_detector_count"], 1) + self.assertEqual(reason, "declared_detector_results_missing") + def test_auto_coder_gate_allows_when_detector_results_confirm_failure(self) -> None: repair_targets = { "targets": [ diff --git a/scripts/test_stage_agent_loop.py b/scripts/test_stage_agent_loop.py index 9e70a47..02ac71f 100644 --- a/scripts/test_stage_agent_loop.py +++ b/scripts/test_stage_agent_loop.py @@ -634,6 +634,13 @@ class StageAgentLoopTests(unittest.TestCase): self.assertTrue(validation["accepted_after_repair"]) self.assertEqual(validation["quality_score"], 92) self.assertEqual(validation["pack_final_status"], "partial") + self.assertEqual(summary["last_quality_score"], 92) + self.assertTrue(summary["last_deterministic_gate_ok"]) + self.assertEqual(summary["last_deterministic_gate_reason"], "deterministic_gate_passed") + self.assertEqual(status["last_quality_score"], 92) + self.assertEqual(status["last_analyst_decision"], "accepted") + self.assertTrue(status["last_deterministic_gate_ok"]) + self.assertEqual(status["last_deterministic_gate_reason"], "deterministic_gate_passed") self.assertEqual(status["latest_validation_source"], "domain_case_loop.run-pack-loop") self.assertEqual(status["latest_replay_validation_status"], "passed") self.assertTrue(status["automation_followup"]["ready_for_next_domain_slice"])