from __future__ import annotations import argparse import json import sys import tempfile import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import agent_runtime_manifest as runtime_manifest def write_text(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") def write_json(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def create_prompt_registry(repo_root: Path, *, preset_version: str = "normalizer_v2_0_2") -> None: write_text( repo_root / "llm_normalizer" / "backend" / "src" / "config.ts", 'export const DEFAULT_PROMPT_VERSION = process.env.DEFAULT_PROMPT_VERSION ?? "normalizer_v2_0_2";\n' "export const FEATURE_ASSISTANT_ADDRESS_QUERY_V1 = toBooleanFlag(\n" " process.env.FEATURE_ASSISTANT_ADDRESS_QUERY_V1,\n" " true\n" ");\n", ) for relative_path in runtime_manifest.BUILTIN_PROMPT_FILES["normalizer_v2_0_2"].values(): write_text(repo_root / "llm_normalizer" / "prompts" / relative_path, f"{relative_path}\n") write_json( repo_root / "llm_normalizer" / "data" / "presets" / "preset-current.json", {"prompt_version": preset_version}, ) class AgentRuntimeManifestTests(unittest.TestCase): def test_prompt_registry_health_passes_for_complete_matching_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo_root = Path(tmp) create_prompt_registry(repo_root) health = runtime_manifest.build_prompt_registry_health(repo_root) self.assertEqual(health["status"], "pass") self.assertEqual(health["prompt_source"], "file") self.assertEqual(health["active_prompt_version"], "normalizer_v2_0_2") self.assertTrue(health["prompt_hash"]) self.assertFalse(health["failures"]) def test_prompt_registry_health_fails_on_preset_mismatch_when_strict(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo_root = Path(tmp) create_prompt_registry(repo_root, preset_version="normalizer_v1") health = runtime_manifest.build_prompt_registry_health(repo_root) self.assertEqual(health["status"], "fail") self.assertTrue(any(str(item).startswith("preset_version_mismatch:") for item in health["failures"])) def test_effective_runtime_manifest_records_runner_and_llm_settings(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo_root = Path(tmp) create_prompt_registry(repo_root) args = argparse.Namespace( backend_url="http://127.0.0.1:8787", mcp_proxy_url="http://127.0.0.1:6003", mcp_channel="default", llm_provider="local", llm_model="test-model", llm_base_url="http://127.0.0.1:1234/v1", temperature=0.0, max_output_tokens=2048, prompt_version="normalizer_v2_0_2", use_mock=False, ) manifest = runtime_manifest.build_effective_runtime_manifest( runner="domain_truth_harness.run-live", args=args, repo_root=repo_root, spec_path=repo_root / "docs" / "orchestration" / "spec.json", output_dir=repo_root / "artifacts" / "domain_runs" / "run", run_id="run", ) self.assertEqual(manifest["runner"], "domain_truth_harness.run-live") self.assertEqual(manifest["llm_model"], "test-model") self.assertEqual(manifest["temperature"], 0.0) self.assertEqual(manifest["max_output_tokens"], 2048) self.assertEqual(manifest["prompt_registry_status"], "pass") def test_effective_runtime_manifest_resolves_address_runtime_prompt_to_default_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo_root = Path(tmp) create_prompt_registry(repo_root) args = argparse.Namespace( backend_url="http://127.0.0.1:8787", mcp_proxy_url="http://127.0.0.1:6003", mcp_channel="default", llm_provider="local", llm_model="test-model", llm_base_url="http://127.0.0.1:1234/v1", temperature=0.0, max_output_tokens=2048, prompt_version="address_query_runtime_v1", use_mock=False, ) manifest = runtime_manifest.build_effective_runtime_manifest( runner="domain_case_loop.run-pack", args=args, repo_root=repo_root, spec_path=repo_root / "docs" / "orchestration" / "spec.json", output_dir=repo_root / "artifacts" / "domain_runs" / "run", run_id="run", ) self.assertEqual(manifest["requested_prompt_version"], "address_query_runtime_v1") self.assertEqual(manifest["assistant_runtime_prompt_version"], "address_query_runtime_v1") self.assertEqual(manifest["prompt_version"], "normalizer_v2_0_2") self.assertEqual(manifest["prompt_resolution"]["mode"], "assistant_runtime_schema_uses_default_normalizer_prompt") self.assertEqual(manifest["prompt_source"], "file") self.assertTrue(manifest["prompt_hash"]) self.assertEqual(manifest["prompt_registry_status"], "pass") def test_load_effective_runtime_manifest_refuses_failing_prompt_registry(self) -> None: with tempfile.TemporaryDirectory() as tmp: run_dir = Path(tmp) runtime_manifest.write_json( run_dir / runtime_manifest.EFFECTIVE_RUNTIME_FILE_NAME, { "schema_version": runtime_manifest.EFFECTIVE_RUNTIME_SCHEMA_VERSION, "git_sha": "test-sha", "runner": "domain_case_loop.run-pack", "llm_model": "test-model", "temperature": 0.0, "max_output_tokens": 2048, "prompt_version": "address_query_runtime_v1", "prompt_source": "unknown", "prompt_hash": None, "prompt_registry_status": "fail", "prompt_registry_failures": ["prompt_hash_unavailable"], }, ) with self.assertRaisesRegex(RuntimeError, "failing prompt registry status|missing prompt_hash"): runtime_manifest.load_effective_runtime_manifest(run_dir) if __name__ == "__main__": unittest.main()