from __future__ import annotations import json import sys import unittest from pathlib import Path import tempfile sys.path.insert(0, str(Path(__file__).resolve().parent)) import agent_runtime_manifest as runtime_manifest import save_agent_semantic_run as saver def write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") class SaveAgentSemanticRunTests(unittest.TestCase): def write_clean_truth_run(self, run_dir: Path, *, include_runtime: bool) -> None: write_json( run_dir / "pack_state.json", { "final_status": "accepted", "review_overall_status": "pass", "acceptance_gate_passed": True, "no_unresolved_p0": True, "unresolved_p0_count": 0, "steps_total": 1, "steps_passed": 1, "steps_failed": 0, }, ) write_json(run_dir / "truth_review.json", {"summary": {"overall_status": "pass"}}) write_json( run_dir / "business_review.json", { "overall_business_status": "pass", "steps_with_business_failures": 0, "steps_with_business_warnings": 0, }, ) if include_runtime: write_json( run_dir / runtime_manifest.EFFECTIVE_RUNTIME_FILE_NAME, { "schema_version": runtime_manifest.EFFECTIVE_RUNTIME_SCHEMA_VERSION, "runner": "domain_truth_harness.run-live", "git_sha": "test-sha", "llm_provider": "local", "llm_model": "test-model", "temperature": 0.0, "max_output_tokens": 2048, "prompt_version": "normalizer_v2_0_2", "prompt_source": "file", "prompt_hash": "abc123", "prompt_registry_status": "pass", }, ) def test_validate_truth_harness_run_refuses_missing_effective_runtime(self) -> None: with tempfile.TemporaryDirectory() as tmp: run_dir = Path(tmp) self.write_clean_truth_run(run_dir, include_runtime=False) with self.assertRaisesRegex(RuntimeError, "reproducibility manifest"): saver.validate_truth_harness_run_dir(run_dir) def test_validate_truth_harness_run_includes_effective_runtime_summary(self) -> None: with tempfile.TemporaryDirectory() as tmp: run_dir = Path(tmp) self.write_clean_truth_run(run_dir, include_runtime=True) metadata = saver.validate_truth_harness_run_dir(run_dir) self.assertEqual(metadata["validation_status"], "accepted_live_replay") self.assertEqual(metadata["effective_runtime"]["runner"], "domain_truth_harness.run-live") self.assertEqual(metadata["effective_runtime"]["llm_model"], "test-model") def test_extract_questions_resolves_scenario_pack_bindings(self) -> None: spec = { "schema_version": "domain_scenario_pack_v1", "bindings": { "main_organization": "ООО Альтернатива Плюс", "control_year": "2020", "svk_counterparty": "Группа СВК", }, "scenarios": [ { "scenario_id": "biz", "steps": [ { "question": "Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год.", "semantic_tags": ["business_overview", "money"], }, { "question": "Отдельно по {{bindings.svk_counterparty}} покажи документы.", "semantic_tags": ["counterparty", "documents"], }, ], } ], } questions = saver.extract_questions_from_spec(spec) self.assertEqual( questions, [ "Дай обзор ООО Альтернатива Плюс за 2020 год.", "Отдельно по Группа СВК покажи документы.", ], ) self.assertFalse(any("{{bindings." in question for question in questions)) self.assertEqual( saver.extract_semantic_tags(spec), ["business_overview", "counterparty", "documents", "money"], ) def test_extract_questions_refuses_unresolved_bindings(self) -> None: spec = { "questions": ["Что с НДС за {{bindings.control_year}} год?"], "bindings": {}, } with self.assertRaisesRegex(RuntimeError, "unresolved bindings"): saver.extract_questions_from_spec(spec) if __name__ == "__main__": unittest.main()