NODEDC_1C/scripts/test_agent_runtime_manifest.py

from __future__ import annotations

import argparse
import json
import sys
import tempfile
import unittest
from pathlib import Path


sys.path.insert(0, str(Path(__file__).resolve().parent))

import agent_runtime_manifest as runtime_manifest


def write_text(path: Path, text: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(text, encoding="utf-8")


def write_json(path: Path, payload: object) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")


def create_prompt_registry(repo_root: Path, *, preset_version: str = "normalizer_v2_0_2") -> None:
    write_text(
        repo_root / "llm_normalizer" / "backend" / "src" / "config.ts",
        'export const DEFAULT_PROMPT_VERSION = process.env.DEFAULT_PROMPT_VERSION ?? "normalizer_v2_0_2";\n'
        "export const FEATURE_ASSISTANT_ADDRESS_QUERY_V1 = toBooleanFlag(\n"
        "  process.env.FEATURE_ASSISTANT_ADDRESS_QUERY_V1,\n"
        "  true\n"
        ");\n",
    )
    for relative_path in runtime_manifest.BUILTIN_PROMPT_FILES["normalizer_v2_0_2"].values():
        write_text(repo_root / "llm_normalizer" / "prompts" / relative_path, f"{relative_path}\n")
    write_json(
        repo_root / "llm_normalizer" / "data" / "presets" / "preset-current.json",
        {"prompt_version": preset_version},
    )


class AgentRuntimeManifestTests(unittest.TestCase):
    def test_prompt_registry_health_passes_for_complete_matching_registry(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            repo_root = Path(tmp)
            create_prompt_registry(repo_root)

            health = runtime_manifest.build_prompt_registry_health(repo_root)

        self.assertEqual(health["status"], "pass")
        self.assertEqual(health["prompt_source"], "file")
        self.assertEqual(health["active_prompt_version"], "normalizer_v2_0_2")
        self.assertTrue(health["prompt_hash"])
        self.assertFalse(health["failures"])

    def test_prompt_registry_health_fails_on_preset_mismatch_when_strict(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            repo_root = Path(tmp)
            create_prompt_registry(repo_root, preset_version="normalizer_v1")

            health = runtime_manifest.build_prompt_registry_health(repo_root)

        self.assertEqual(health["status"], "fail")
        self.assertTrue(any(str(item).startswith("preset_version_mismatch:") for item in health["failures"]))

    def test_effective_runtime_manifest_records_runner_and_llm_settings(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            repo_root = Path(tmp)
            create_prompt_registry(repo_root)
            args = argparse.Namespace(
                backend_url="http://127.0.0.1:8787",
                mcp_proxy_url="http://127.0.0.1:6003",
                mcp_channel="default",
                llm_provider="local",
                llm_model="test-model",
                llm_base_url="http://127.0.0.1:1234/v1",
                temperature=0.0,
                max_output_tokens=2048,
                prompt_version="normalizer_v2_0_2",
                use_mock=False,
            )

            manifest = runtime_manifest.build_effective_runtime_manifest(
                runner="domain_truth_harness.run-live",
                args=args,
                repo_root=repo_root,
                spec_path=repo_root / "docs" / "orchestration" / "spec.json",
                output_dir=repo_root / "artifacts" / "domain_runs" / "run",
                run_id="run",
            )

        self.assertEqual(manifest["runner"], "domain_truth_harness.run-live")
        self.assertEqual(manifest["llm_model"], "test-model")
        self.assertEqual(manifest["temperature"], 0.0)
        self.assertEqual(manifest["max_output_tokens"], 2048)
        self.assertEqual(manifest["prompt_registry_status"], "pass")

    def test_effective_runtime_manifest_resolves_address_runtime_prompt_to_default_registry(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            repo_root = Path(tmp)
            create_prompt_registry(repo_root)
            args = argparse.Namespace(
                backend_url="http://127.0.0.1:8787",
                mcp_proxy_url="http://127.0.0.1:6003",
                mcp_channel="default",
                llm_provider="local",
                llm_model="test-model",
                llm_base_url="http://127.0.0.1:1234/v1",
                temperature=0.0,
                max_output_tokens=2048,
                prompt_version="address_query_runtime_v1",
                use_mock=False,
            )

            manifest = runtime_manifest.build_effective_runtime_manifest(
                runner="domain_case_loop.run-pack",
                args=args,
                repo_root=repo_root,
                spec_path=repo_root / "docs" / "orchestration" / "spec.json",
                output_dir=repo_root / "artifacts" / "domain_runs" / "run",
                run_id="run",
            )

        self.assertEqual(manifest["requested_prompt_version"], "address_query_runtime_v1")
        self.assertEqual(manifest["assistant_runtime_prompt_version"], "address_query_runtime_v1")
        self.assertEqual(manifest["prompt_version"], "normalizer_v2_0_2")
        self.assertEqual(manifest["prompt_resolution"]["mode"], "assistant_runtime_schema_uses_default_normalizer_prompt")
        self.assertEqual(manifest["prompt_source"], "file")
        self.assertTrue(manifest["prompt_hash"])
        self.assertEqual(manifest["prompt_registry_status"], "pass")

    def test_load_effective_runtime_manifest_refuses_failing_prompt_registry(self) -> None:
        with tempfile.TemporaryDirectory() as tmp:
            run_dir = Path(tmp)
            runtime_manifest.write_json(
                run_dir / runtime_manifest.EFFECTIVE_RUNTIME_FILE_NAME,
                {
                    "schema_version": runtime_manifest.EFFECTIVE_RUNTIME_SCHEMA_VERSION,
                    "git_sha": "test-sha",
                    "runner": "domain_case_loop.run-pack",
                    "llm_model": "test-model",
                    "temperature": 0.0,
                    "max_output_tokens": 2048,
                    "prompt_version": "address_query_runtime_v1",
                    "prompt_source": "unknown",
                    "prompt_hash": None,
                    "prompt_registry_status": "fail",
                    "prompt_registry_failures": ["prompt_hash_unavailable"],
                },
            )

            with self.assertRaisesRegex(RuntimeError, "failing prompt registry status|missing prompt_hash"):
                runtime_manifest.load_effective_runtime_manifest(run_dir)


if __name__ == "__main__":
    unittest.main()