134 lines
5.0 KiB
Python
134 lines
5.0 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
import unittest
|
||
from pathlib import Path
|
||
import tempfile
|
||
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
|
||
import agent_runtime_manifest as runtime_manifest
|
||
import save_agent_semantic_run as saver
|
||
|
||
|
||
def write_json(path: Path, payload: object) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
|
||
class SaveAgentSemanticRunTests(unittest.TestCase):
|
||
def write_clean_truth_run(self, run_dir: Path, *, include_runtime: bool) -> None:
|
||
write_json(
|
||
run_dir / "pack_state.json",
|
||
{
|
||
"final_status": "accepted",
|
||
"review_overall_status": "pass",
|
||
"acceptance_gate_passed": True,
|
||
"no_unresolved_p0": True,
|
||
"unresolved_p0_count": 0,
|
||
"steps_total": 1,
|
||
"steps_passed": 1,
|
||
"steps_failed": 0,
|
||
},
|
||
)
|
||
write_json(run_dir / "truth_review.json", {"summary": {"overall_status": "pass"}})
|
||
write_json(
|
||
run_dir / "business_review.json",
|
||
{
|
||
"overall_business_status": "pass",
|
||
"steps_with_business_failures": 0,
|
||
"steps_with_business_warnings": 0,
|
||
},
|
||
)
|
||
if include_runtime:
|
||
write_json(
|
||
run_dir / runtime_manifest.EFFECTIVE_RUNTIME_FILE_NAME,
|
||
{
|
||
"schema_version": runtime_manifest.EFFECTIVE_RUNTIME_SCHEMA_VERSION,
|
||
"runner": "domain_truth_harness.run-live",
|
||
"git_sha": "test-sha",
|
||
"llm_provider": "local",
|
||
"llm_model": "test-model",
|
||
"temperature": 0.0,
|
||
"max_output_tokens": 2048,
|
||
"prompt_version": "normalizer_v2_0_2",
|
||
"prompt_source": "file",
|
||
"prompt_hash": "abc123",
|
||
"prompt_registry_status": "pass",
|
||
},
|
||
)
|
||
|
||
def test_validate_truth_harness_run_refuses_missing_effective_runtime(self) -> None:
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
run_dir = Path(tmp)
|
||
self.write_clean_truth_run(run_dir, include_runtime=False)
|
||
|
||
with self.assertRaisesRegex(RuntimeError, "reproducibility manifest"):
|
||
saver.validate_truth_harness_run_dir(run_dir)
|
||
|
||
def test_validate_truth_harness_run_includes_effective_runtime_summary(self) -> None:
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
run_dir = Path(tmp)
|
||
self.write_clean_truth_run(run_dir, include_runtime=True)
|
||
|
||
metadata = saver.validate_truth_harness_run_dir(run_dir)
|
||
|
||
self.assertEqual(metadata["validation_status"], "accepted_live_replay")
|
||
self.assertEqual(metadata["effective_runtime"]["runner"], "domain_truth_harness.run-live")
|
||
self.assertEqual(metadata["effective_runtime"]["llm_model"], "test-model")
|
||
|
||
def test_extract_questions_resolves_scenario_pack_bindings(self) -> None:
|
||
spec = {
|
||
"schema_version": "domain_scenario_pack_v1",
|
||
"bindings": {
|
||
"main_organization": "ООО Альтернатива Плюс",
|
||
"control_year": "2020",
|
||
"svk_counterparty": "Группа СВК",
|
||
},
|
||
"scenarios": [
|
||
{
|
||
"scenario_id": "biz",
|
||
"steps": [
|
||
{
|
||
"question": "Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год.",
|
||
"semantic_tags": ["business_overview", "money"],
|
||
},
|
||
{
|
||
"question": "Отдельно по {{bindings.svk_counterparty}} покажи документы.",
|
||
"semantic_tags": ["counterparty", "documents"],
|
||
},
|
||
],
|
||
}
|
||
],
|
||
}
|
||
|
||
questions = saver.extract_questions_from_spec(spec)
|
||
|
||
self.assertEqual(
|
||
questions,
|
||
[
|
||
"Дай обзор ООО Альтернатива Плюс за 2020 год.",
|
||
"Отдельно по Группа СВК покажи документы.",
|
||
],
|
||
)
|
||
self.assertFalse(any("{{bindings." in question for question in questions))
|
||
self.assertEqual(
|
||
saver.extract_semantic_tags(spec),
|
||
["business_overview", "counterparty", "documents", "money"],
|
||
)
|
||
|
||
def test_extract_questions_refuses_unresolved_bindings(self) -> None:
|
||
spec = {
|
||
"questions": ["Что с НДС за {{bindings.control_year}} год?"],
|
||
"bindings": {},
|
||
}
|
||
|
||
with self.assertRaisesRegex(RuntimeError, "unresolved bindings"):
|
||
saver.extract_questions_from_spec(spec)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|