120 lines
5.1 KiB
Python
120 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
import domain_case_loop as dcl
|
|
import domain_truth_harness as dth
|
|
|
|
|
|
class DomainCaseLoopStepStateTests(unittest.TestCase):
|
|
def test_preserves_mcp_catalog_alignment_debug_fields(self) -> None:
|
|
step_state = dcl.build_scenario_step_state(
|
|
scenario_id="planner_alignment_demo",
|
|
domain="planner_autonomy",
|
|
step={
|
|
"step_id": "step_01",
|
|
"title": "Alignment visibility",
|
|
"depends_on": [],
|
|
"question_template": "show planner alignment",
|
|
},
|
|
step_index=1,
|
|
question_resolved="show planner alignment",
|
|
analysis_context={},
|
|
turn_artifact={
|
|
"assistant_message": {
|
|
"reply_type": "factual",
|
|
"text": "Confirmed answer",
|
|
"message_id": "msg-1",
|
|
"trace_id": "trace-1",
|
|
},
|
|
"technical_debug_payload": {
|
|
"detected_mode": "address_query",
|
|
"detected_intent": "counterparty_turnover",
|
|
"selected_recipe": "counterparty_turnover_by_period",
|
|
"capability_id": "confirmed_counterparty_turnover",
|
|
"mcp_discovery_catalog_chain_alignment_status": "selected_matches_top",
|
|
"mcp_discovery_catalog_chain_top_match": "value_flow",
|
|
"mcp_discovery_catalog_chain_selected_matches_top": True,
|
|
},
|
|
"session_summary": {},
|
|
},
|
|
entries=[],
|
|
)
|
|
|
|
self.assertEqual(step_state["mcp_discovery_catalog_chain_alignment_status"], "selected_matches_top")
|
|
self.assertEqual(step_state["mcp_discovery_catalog_chain_top_match"], "value_flow")
|
|
self.assertTrue(step_state["mcp_discovery_catalog_chain_selected_matches_top"])
|
|
|
|
def test_truth_harness_warns_on_catalog_alignment_divergence(self) -> None:
|
|
reviewed = dth.evaluate_truth_step(
|
|
step={
|
|
"step_id": "step_01",
|
|
"question_template": "show planner alignment",
|
|
"criticality": "critical",
|
|
"allowed_reply_types": [],
|
|
},
|
|
step_state={
|
|
"question_resolved": "show planner alignment",
|
|
"reply_type": "factual",
|
|
"assistant_text": "Confirmed answer",
|
|
"actual_direct_answer": "Confirmed answer",
|
|
"detected_intent": "counterparty_turnover",
|
|
"selected_recipe": "counterparty_turnover_by_period",
|
|
"capability_id": "confirmed_counterparty_turnover",
|
|
"mcp_discovery_catalog_chain_alignment_status": "selected_outside_match_set",
|
|
"mcp_discovery_catalog_chain_top_match": "value_flow_comparison",
|
|
"mcp_discovery_catalog_chain_selected_matches_top": False,
|
|
"extracted_filters": {},
|
|
},
|
|
step_results={},
|
|
bindings={},
|
|
runtime_bindings={},
|
|
)
|
|
|
|
self.assertEqual(reviewed["review_status"], "warning")
|
|
self.assertEqual(reviewed["warning_findings_count"], 1)
|
|
self.assertEqual(reviewed["review_findings"][0]["code"], "catalog_alignment_divergence")
|
|
self.assertEqual(reviewed["review_findings"][0]["severity"], "warning")
|
|
|
|
def test_truth_harness_checks_expected_catalog_alignment_fields(self) -> None:
|
|
reviewed = dth.evaluate_truth_step(
|
|
step={
|
|
"step_id": "step_01",
|
|
"question_template": "show planner alignment",
|
|
"criticality": "critical",
|
|
"allowed_reply_types": [],
|
|
"expected_catalog_alignment_status": "selected_matches_top",
|
|
"expected_catalog_chain_top_match": "value_flow_comparison",
|
|
"expected_catalog_selected_matches_top": True,
|
|
},
|
|
step_state={
|
|
"question_resolved": "show planner alignment",
|
|
"reply_type": "factual",
|
|
"assistant_text": "Confirmed answer",
|
|
"actual_direct_answer": "Confirmed answer",
|
|
"detected_intent": "counterparty_turnover",
|
|
"selected_recipe": "counterparty_turnover_by_period",
|
|
"capability_id": "confirmed_counterparty_turnover",
|
|
"mcp_discovery_catalog_chain_alignment_status": "selected_matches_top",
|
|
"mcp_discovery_catalog_chain_top_match": "value_flow",
|
|
"mcp_discovery_catalog_chain_selected_matches_top": True,
|
|
"extracted_filters": {},
|
|
},
|
|
step_results={},
|
|
bindings={},
|
|
runtime_bindings={},
|
|
)
|
|
|
|
self.assertEqual(reviewed["review_status"], "fail")
|
|
self.assertEqual(reviewed["critical_findings_count"], 1)
|
|
self.assertEqual(reviewed["review_findings"][0]["code"], "wrong_catalog_chain_top_match")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|