From ef8391e06997045560e17677caaeafca67e9fed4 Mon Sep 17 00:00:00 2001 From: dctouch Date: Wed, 3 Jun 2026 13:54:06 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=BE=D1=80=D0=BC=D0=B0=D0=BB=D0=B8?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D1=8E=D1=80=D0=BB=D0=B8?= =?UTF-8?q?=D1=86=D0=B0=20=D0=B2=20truth-harness=20=D1=84=D0=B8=D0=BB?= =?UTF-8?q?=D1=8C=D1=82=D1=80=D0=B0=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/domain_truth_harness.py | 7 ++- scripts/test_domain_truth_harness.py | 80 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 scripts/test_domain_truth_harness.py diff --git a/scripts/domain_truth_harness.py b/scripts/domain_truth_harness.py index b24f41a..dcc4241 100644 --- a/scripts/domain_truth_harness.py +++ b/scripts/domain_truth_harness.py @@ -976,7 +976,7 @@ def evaluate_truth_step( expected=expected_value, ) continue - if actual_value != expected_value: + if not dcl.validation_filter_values_match(filter_key, actual_value, expected_value): append_finding( findings, step, @@ -1006,7 +1006,10 @@ def evaluate_truth_step( ) for filter_key, forbidden_values in forbidden_filter_values.items(): actual_value = normalize_actual_filter_value(filter_key, extracted_filters.get(filter_key)) - if actual_value and actual_value in forbidden_values: + if actual_value and any( + dcl.validation_filter_values_match(filter_key, actual_value, forbidden_value) + for forbidden_value in forbidden_values + ): append_finding( findings, step, diff --git a/scripts/test_domain_truth_harness.py b/scripts/test_domain_truth_harness.py new file mode 100644 index 0000000..3fb77c5 --- /dev/null +++ b/scripts/test_domain_truth_harness.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import domain_truth_harness as dth + + +class DomainTruthHarnessTests(unittest.TestCase): + def test_required_organization_filter_accepts_quoted_legal_name(self) -> None: + step = dth.normalize_step_spec( + 1, + { + "step_id": "step_01", + "title": "Inventory root", + "question_template": "что там на складе по остаткам", + "required_filters": {"organization": "ООО Альтернатива Плюс"}, + }, + ) + reviewed = dth.evaluate_truth_step( + step=step, + step_state={ + "question_resolved": "что там на складе по остаткам", + "reply_type": "factual", + "assistant_text": "На складе подтверждено 11 позиций.", + "extracted_filters": {"organization": 'ООО "Альтернатива Плюс"'}, + }, + step_results={}, + bindings={}, + runtime_bindings={}, + ) + + self.assertEqual(reviewed["review_status"], "pass") + self.assertFalse( + [ + finding + for finding in reviewed["review_findings"] + if finding.get("code") == "wrong_filter:organization" + ] + ) + + def test_forbidden_organization_filter_matches_quoted_legal_name(self) -> None: + step = dth.normalize_step_spec( + 1, + { + "step_id": "step_01", + "title": "Wrong organization scope", + "question_template": "что там на складе по остаткам", + "forbidden_filter_values": {"organization": ["ООО Альтернатива Плюс"]}, + }, + ) + reviewed = dth.evaluate_truth_step( + step=step, + step_state={ + "question_resolved": "что там на складе по остаткам", + "reply_type": "factual", + "assistant_text": "На складе подтверждено 11 позиций.", + "extracted_filters": {"organization": 'ООО "Альтернатива Плюс"'}, + }, + step_results={}, + bindings={}, + runtime_bindings={}, + ) + + self.assertEqual(reviewed["review_status"], "fail") + self.assertTrue( + [ + finding + for finding in reviewed["review_findings"] + if finding.get("code") == "forbidden_filter_value:organization" + ] + ) + + +if __name__ == "__main__": + unittest.main()