SILVER
/
NODEDC_1C
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Усилить агентный detector gate для declared detectors

This commit is contained in:
dctouch 2026-06-15 17:18:58 +03:00
parent bebea0361b
commit 9e2bc8cb6c
7 changed files with 234 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/orchestration/detector_registry.json
View File

@ -65,6 +65,12 @@
"check": {
"required_patterns_any": [
"(?i)(выруч|себестоим|валов|марж|не хватает|не могу подтвердить|не подтвержден|unknown)"
],
"artifact_path_exclude_patterns": [
"(?i)selected_item",
"(?i)supplier",
"(?i)purchase_date",
"(?i)purchase_document"
]
}
},

20
scripts/agent_detector_runner.py
View File

@ -295,6 +295,22 @@ def filter_turns(turns: list[dict[str, Any]], artifact_dir: Path, evidence_paths
return [item for item in turns if path_matches_evidence(item["path"], artifact_dir, evidence_paths)]
def filter_artifacts_by_detector_scope(items: list[dict[str, Any]], check: dict[str, Any]) -> list[dict[str, Any]]:
include_patterns = compile_patterns(normalize_string_list(check.get("artifact_path_include_patterns")))
exclude_patterns = compile_patterns(normalize_string_list(check.get("artifact_path_exclude_patterns")))
if not include_patterns and not exclude_patterns:
return items
filtered: list[dict[str, Any]] = []
for item in items:
artifact_path = str(item.get("artifact_path") or item.get("repo_path") or "")
if include_patterns and not any(pattern.search(artifact_path) for pattern in include_patterns):
continue
if exclude_patterns and any(pattern.search(artifact_path) for pattern in exclude_patterns):
continue
filtered.append(item)
return filtered
def compile_patterns(patterns: list[str]) -> list[re.Pattern[str]]:
compiled: list[re.Pattern[str]] = []
for pattern in patterns:
@ -591,8 +607,8 @@ def evaluate_detector(
) -> dict[str, Any]:
kind = str(detector.get("kind") or "").strip()
check = detector.get("check") if isinstance(detector.get("check"), dict) else {}
scoped_outputs = filter_outputs(outputs, artifact_dir, evidence_paths)
scoped_turns = filter_turns(turns, artifact_dir, evidence_paths)
scoped_outputs = filter_artifacts_by_detector_scope(filter_outputs(outputs, artifact_dir, evidence_paths), check)
scoped_turns = filter_artifacts_by_detector_scope(filter_turns(turns, artifact_dir, evidence_paths), check)
if kind == "artifact_presence":
return evaluate_artifact_presence(detector_name, detector, artifact_dir)
if kind == "answer_text_regex_forbidden":

98
scripts/domain_case_loop.py
View File

@ -6107,11 +6107,45 @@ def build_accepted_pack_smoke_detector_candidates() -> list[dict[str, Any]]:
]
def build_detector_candidates(repair_targets: dict[str, Any], catalog: dict[str, Any] | None = None) -> dict[str, Any]:
def build_detector_candidates(
repair_targets: dict[str, Any],
catalog: dict[str, Any] | None = None,
*,
declared_detectors: list[str] | None = None,
declared_issue_codes: list[str] | None = None,
) -> dict[str, Any]:
source = catalog if isinstance(catalog, dict) else load_issue_catalog()
issues = source.get("issues") if isinstance(source.get("issues"), dict) else {}
candidates: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
def add_candidate(
*,
issue_code: str,
detector: str,
severity: Any = None,
sample_target_id: str | None = None,
evidence_paths: list[str] | None = None,
source_label: str,
) -> None:
detector_name = str(detector or "").strip()
if not detector_name:
return
key = (str(issue_code or "").strip(), detector_name)
if key in seen:
return
seen.add(key)
candidates.append(
{
"issue_code": key[0],
"detector": detector_name,
"severity": severity,
"sample_target_id": sample_target_id,
"evidence_paths": evidence_paths or [],
"source": source_label,
}
)
for target in repair_targets.get("targets") if isinstance(repair_targets.get("targets"), list) else []:
if not isinstance(target, dict):
continue
@ -6122,24 +6156,42 @@ def build_detector_candidates(repair_targets: dict[str, Any], catalog: dict[str,
detectors = [f"{issue_code}_detector"]
evidence_paths = detector_evidence_paths_for_target(target)
for detector in detectors:
key = (issue_code, detector)
if key in seen:
continue
seen.add(key)
candidates.append(
{
"issue_code": issue_code,
"detector": detector,
"severity": target.get("severity"),
"sample_target_id": target.get("target_id"),
"evidence_paths": evidence_paths,
}
add_candidate(
issue_code=issue_code,
detector=detector,
severity=target.get("severity"),
sample_target_id=target.get("target_id"),
evidence_paths=evidence_paths,
source_label="repair_target",
)
declared_codes = normalize_string_list(declared_issue_codes)
detector_to_declared_issue: dict[str, str] = {}
for issue_code in declared_codes:
entry = issues.get(issue_code) if isinstance(issues.get(issue_code), dict) else {}
for detector in normalize_string_list(entry.get("detectors")):
detector_to_declared_issue.setdefault(detector, issue_code)
add_candidate(
issue_code=issue_code,
detector=detector,
severity=entry.get("severity"),
sample_target_id="manifest.issue_codes_under_test",
source_label="manifest.issue_codes_under_test",
)
for detector in normalize_string_list(declared_detectors):
issue_code = detector_to_declared_issue.get(detector, "")
add_candidate(
issue_code=issue_code,
detector=detector,
sample_target_id="manifest.detectors_under_test",
source_label="manifest.detectors_under_test",
)
if should_add_accepted_pack_smoke_detectors(repair_targets, candidates):
candidates.extend(build_accepted_pack_smoke_detector_candidates())
return {
"schema_version": "detector_candidates_v1",
"candidate_count": len(candidates),
"declared_detector_count": len(normalize_string_list(declared_detectors)),
"declared_issue_code_count": len(declared_codes),
"candidates": candidates,
}
@ -6149,6 +6201,7 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit
return {
"status": "not_run",
"detector_count": 0,
"declared_detector_count": 0,
"pass": 0,
"fail": 0,
"review": 0,
@ -6172,9 +6225,11 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit
return names[:limit]
status = str(summary.get("status") or "skipped")
declared_detector_count = len(normalize_string_list(detector_results.get("declared_detectors_under_test")))
return {
"status": status,
"detector_count": int(summary.get("detector_count") or len(results)),
"declared_detector_count": declared_detector_count,
"pass": int(summary.get("pass") or 0),
"fail": int(summary.get("fail") or 0),
"review": int(summary.get("review") or 0),
@ -6188,6 +6243,12 @@ def summarize_detector_results(detector_results: dict[str, Any] | None, *, limit
def evaluate_detector_results_gate(detector_results_summary: dict[str, Any]) -> tuple[bool, str]:
status = str(detector_results_summary.get("status") or "not_run").strip()
declared_detector_count = int(detector_results_summary.get("declared_detector_count") or 0)
detector_count = int(detector_results_summary.get("detector_count") or 0)
if declared_detector_count > 0 and detector_count == 0:
return False, "declared_detector_results_missing"
if declared_detector_count > 0 and status == "skipped":
return False, "declared_detector_results_skipped"
if status == "fail":
failed = normalize_string_list(detector_results_summary.get("failed_detectors"))
suffix = ",".join(failed) if failed else str(detector_results_summary.get("fail") or 0)
@ -6894,6 +6955,7 @@ def build_loop_final_status(loop_state: dict[str, Any]) -> str:
def handle_run_pack_loop(args: argparse.Namespace) -> int:
manifest_path = Path(args.manifest).resolve()
pack_manifest = read_json_file(manifest_path)
loop_id = str(args.loop_id or slugify_case_id("domain_pack_loop", None)).strip()
loop_dir = Path(args.output_root).resolve() / loop_id
iterations_dir = loop_dir / "iterations"
@ -7026,13 +7088,21 @@ def handle_run_pack_loop(args: argparse.Namespace) -> int:
"source_repair_targets": repo_relative(repair_targets_path),
"items": collect_rerun_matrix(repair_targets),
}
detector_candidates = build_detector_candidates(repair_targets)
declared_detectors = normalize_string_list(pack_manifest.get("detectors_under_test"))
declared_issue_codes = normalize_string_list(pack_manifest.get("issue_codes_under_test"))
detector_candidates = build_detector_candidates(
repair_targets,
declared_detectors=declared_detectors,
declared_issue_codes=declared_issue_codes,
)
write_json(detector_candidates_path, detector_candidates)
detector_results = agent_detector_runner.build_detector_results(
pack_dir,
detector_candidates_path=detector_candidates_path,
include_default_global=False,
)
detector_results["declared_detectors_under_test"] = declared_detectors
detector_results["declared_issue_codes_under_test"] = declared_issue_codes
write_json(detector_results_path, detector_results)
detector_results_summary = summarize_detector_results(detector_results)
detector_results_gate_ok, detector_results_gate_reason = evaluate_detector_results_gate(detector_results_summary)

14
scripts/stage_agent_loop.py
View File

@ -1830,6 +1830,16 @@ def build_replay_validation_stage_summary(
"global_plan_refs": stage_manifest.get("global_plan_refs") or base.get("global_plan_refs") or [],
"target_score": stage_manifest.get("target_score", base.get("target_score")),
"acceptance_invariants": stage_manifest.get("acceptance_invariants") or base.get("acceptance_invariants") or [],
"loop_final_status": validation.get("final_status") or base.get("loop_final_status"),
"stop_reason": "accepted_validation_ingested" if accepted else "validation_not_accepted",
"iterations_ran": base.get("iterations_ran") or (1 if validation.get("last_iteration_id") else None),
"last_quality_score": validation.get("quality_score", base.get("last_quality_score")),
"last_analyst_decision": validation.get("review_overall_status") or base.get("last_analyst_decision"),
"last_deterministic_gate_ok": validation.get("deterministic_gate_ok", base.get("last_deterministic_gate_ok")),
"last_deterministic_gate_reason": validation.get(
"deterministic_gate_reason",
base.get("last_deterministic_gate_reason"),
),
"accepted_gate": accepted,
"manual_confirmation_required": accepted,
"next_action": next_action,
@ -2608,6 +2618,10 @@ def build_stage_status(stage_manifest: dict[str, Any], stage_dir: Path) -> dict[
"summary_exists": bool(summary),
"repair_mode": repair_mode,
"loop_final_status": summary.get("loop_final_status"),
"last_quality_score": summary.get("last_quality_score"),
"last_analyst_decision": summary.get("last_analyst_decision"),
"last_deterministic_gate_ok": summary.get("last_deterministic_gate_ok"),
"last_deterministic_gate_reason": summary.get("last_deterministic_gate_reason"),
"accepted_gate": summary.get("accepted_gate"),
"loop_accepted_gate": summary.get("loop_accepted_gate"),
"stage_closing_gate": stage_closing_gate or None,

54
scripts/test_agent_detector_runner.py
View File

@ -166,6 +166,60 @@ class AgentDetectorRunnerTests(unittest.TestCase):
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
def test_required_any_detector_can_exclude_selected_object_followup_paths(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
artifact_dir = root / "run"
write_text(
artifact_dir / "scenarios" / "margin_root" / "steps" / "step_01" / "output.md",
"Top margin item: revenue 100, cogs 60, gross profit 40, margin 40%.",
)
write_text(
artifact_dir
/ "scenarios"
/ "margin_followup"
/ "steps"
/ "step_03_selected_item_supplier"
/ "output.md",
"Supplier for the selected item: Example LLC.",
)
registry_path = root / "detector_registry.json"
issue_catalog_path = root / "issue_catalog.json"
write_json(
registry_path,
{
"schema_version": "agent_detector_registry_v1",
"detectors": {
"missing_revenue_cogs_margin_fields": {
"kind": "answer_text_required_any",
"automation_level": "semi_automatic",
"description": "Root margin answers need margin fields.",
"issue_codes": ["margin_domain_leak_accounting_route"],
"inputs": ["output.md"],
"check": {
"required_patterns_any": ["(?i)(revenue|cogs|gross profit|margin)"],
"artifact_path_exclude_patterns": ["(?i)selected_item"],
},
}
},
},
)
write_json(issue_catalog_path, {"schema_version": "agent_issue_catalog_v1", "issues": {}})
results = runner.build_detector_results(
artifact_dir,
detector_names=["missing_revenue_cogs_margin_fields"],
registry_path=registry_path,
issue_catalog_path=issue_catalog_path,
include_default_global=False,
)
self.assertEqual(results["summary"]["status"], "pass")
self.assertEqual(results["results"][0]["status"], "pass")
evidence_paths = [item["path"] for item in results["results"][0]["evidence"]]
self.assertEqual(len(evidence_paths), 1)
self.assertIn("margin_root", evidence_paths[0])
def test_composite_detector_fails_after_child_detector_fails(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)

51
scripts/test_domain_case_loop_lead_handoff.py
View File

@ -353,6 +353,57 @@ class DomainCaseLoopLeadHandoffTests(unittest.TestCase):
self.assertEqual(gate["detector_results_summary"]["status"], "pass")
self.assertIn("detector_results_no_repair_signal:pass", gate["blocking_reasons"])
def test_detector_candidates_include_manifest_declared_detectors_without_repair_targets(self) -> None:
repair_targets = {
"target_count": 0,
"acceptance_status": "accepted",
"final_status": "accepted",
"targets": [],
}
catalog = {
"issues": {
"technical_garbage_in_answer": {
"severity": "P0",
"detectors": ["runtime_tokens_in_user_answer", "capability_ids_in_user_answer"],
},
"margin_domain_leak_accounting_route": {
"severity": "P0",
"detectors": ["forbidden_margin_terms"],
},
}
}
candidates = dcl.build_detector_candidates(
repair_targets,
catalog=catalog,
declared_issue_codes=["technical_garbage_in_answer"],
declared_detectors=["forbidden_margin_terms"],
)
detector_names = [item["detector"] for item in candidates["candidates"]]
source_by_detector = {item["detector"]: item["source"] for item in candidates["candidates"]}
self.assertEqual(candidates["declared_detector_count"], 1)
self.assertEqual(candidates["declared_issue_code_count"], 1)
self.assertIn("runtime_tokens_in_user_answer", detector_names)
self.assertIn("capability_ids_in_user_answer", detector_names)
self.assertIn("forbidden_margin_terms", detector_names)
self.assertEqual(source_by_detector["forbidden_margin_terms"], "manifest.detectors_under_test")
def test_detector_gate_blocks_empty_run_when_manifest_declares_detectors(self) -> None:
summary = dcl.summarize_detector_results(
{
"summary": {"status": "skipped", "detector_count": 0, "pass": 0, "fail": 0, "review": 0, "skipped": 0},
"results": [],
"declared_detectors_under_test": ["forbidden_margin_terms"],
}
)
ok, reason = dcl.evaluate_detector_results_gate(summary)
self.assertFalse(ok)
self.assertEqual(summary["declared_detector_count"], 1)
self.assertEqual(reason, "declared_detector_results_missing")
def test_auto_coder_gate_allows_when_detector_results_confirm_failure(self) -> None:
repair_targets = {
"targets": [

7
scripts/test_stage_agent_loop.py
View File

@ -634,6 +634,13 @@ class StageAgentLoopTests(unittest.TestCase):
self.assertTrue(validation["accepted_after_repair"])
self.assertEqual(validation["quality_score"], 92)
self.assertEqual(validation["pack_final_status"], "partial")
self.assertEqual(summary["last_quality_score"], 92)
self.assertTrue(summary["last_deterministic_gate_ok"])
self.assertEqual(summary["last_deterministic_gate_reason"], "deterministic_gate_passed")
self.assertEqual(status["last_quality_score"], 92)
self.assertEqual(status["last_analyst_decision"], "accepted")
self.assertTrue(status["last_deterministic_gate_ok"])
self.assertEqual(status["last_deterministic_gate_reason"], "deterministic_gate_passed")
self.assertEqual(status["latest_validation_source"], "domain_case_loop.run-pack-loop")
self.assertEqual(status["latest_replay_validation_status"], "passed")
self.assertTrue(status["automation_followup"]["ready_for_next_domain_slice"])