Сохранить HM VAT old-stock AGENT pack без mojibake

scripts/save_agent_semantic_run.py

@@ -17,6 +17,7 @@ SAVED_SESSIONS_DIR = REPO_ROOT / "llm_normalizer" / "data" / "autorun_generators
 EVAL_CASES_DIR = REPO_ROOT / "llm_normalizer" / "data" / "eval_cases"
 VALIDATED_AGENT_SAVE_SCHEMA_VERSION = "agent_semantic_save_gate_v1"
 BINDING_TOKEN_RE = re.compile(r"\{\{\s*bindings\.([A-Za-z0-9_-]+)\s*\}\}")
+MOJIBAKE_MARKERS = ("Р", "С", "Ð", "Ñ")
 
 
 def now_utc() -> datetime:
@@ -38,7 +39,34 @@ def sanitize_question(value: Any) -> str:
     text = str(value or "").replace("\r\n", "\n").replace("\r", "\n")
     text = "\n".join(line.strip() for line in text.split("\n"))
     text = re.sub(r"[ \t]+", " ", text).strip()
-    return text
+    return repair_text_mojibake(text)
+
+
+def mojibake_score(value: str) -> int:
+    if not value:
+        return 0
+    score = sum(value.count(marker) for marker in MOJIBAKE_MARKERS)
+    score += value.count("\ufffd") * 5
+    score += sum(5 for char in value if "\u2500" <= char <= "\u257f")
+    return score
+
+
+def repair_text_mojibake(value: str) -> str:
+    if not value:
+        return value
+    best = value
+    best_score = mojibake_score(value)
+    for source_encoding in ("latin1", "cp1251"):
+        for target_encoding in ("utf-8", "cp1251", "cp866"):
+            try:
+                repaired = value.encode(source_encoding).decode(target_encoding)
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                continue
+            repaired_score = mojibake_score(repaired)
+            if repaired_score < best_score:
+                best = repaired
+                best_score = repaired_score
+    return best
 
 
 def normalize_bindings(raw_bindings: Any) -> dict[str, str]:
@@ -47,7 +75,7 @@ def normalize_bindings(raw_bindings: Any) -> dict[str, str]:
     result: dict[str, str] = {}
     for key, value in raw_bindings.items():
         normalized_key = str(key or "").strip()
-        normalized_value = str(value or "").strip()
+        normalized_value = repair_text_mojibake(str(value or "").strip())
         if normalized_key and normalized_value:
             result[normalized_key] = normalized_value
     return result

scripts/test_save_agent_semantic_run.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import json
 import sys
+import tempfile
 import unittest
 from pathlib import Path
-import tempfile
 
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -13,6 +13,10 @@ import agent_runtime_manifest as runtime_manifest
 import save_agent_semantic_run as saver
 
 
+def mojibake(value: str) -> str:
+    return value.encode("utf-8").decode("cp1251")
+
+
 def write_json(path: Path, payload: object) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -139,20 +143,20 @@ class SaveAgentSemanticRunTests(unittest.TestCase):
         spec = {
             "schema_version": "domain_scenario_pack_v1",
             "bindings": {
-                "main_organization": "ООО Альтернатива Плюс",
+                "main_organization": mojibake("ООО Альтернатива Плюс"),
                 "control_year": "2020",
-                "svk_counterparty": "Группа СВК",
+                "svk_counterparty": mojibake("Группа СВК"),
             },
             "scenarios": [
                 {
                     "scenario_id": "biz",
                     "steps": [
                         {
-                            "question": "Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год.",
+                            "question": mojibake("Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год."),
                             "semantic_tags": ["business_overview", "money"],
                         },
                         {
-                            "question": "Отдельно по {{bindings.svk_counterparty}} покажи документы.",
+                            "question": mojibake("Отдельно по {{bindings.svk_counterparty}} покажи документы."),
                             "semantic_tags": ["counterparty", "documents"],
                         },
                     ],
@@ -175,6 +179,36 @@ class SaveAgentSemanticRunTests(unittest.TestCase):
             ["business_overview", "counterparty", "documents", "money"],
         )
 
+    def test_extract_questions_repairs_mojibake_before_saved_session(self) -> None:
+        spec = {
+            "schema_version": "domain_truth_harness_spec_v1",
+            "bindings": {
+                "organization": mojibake('ООО "Альтернатива Плюс"'),
+            },
+            "steps": [
+                {
+                    "question": mojibake("Какие остатки на складе у {{bindings.organization}}?"),
+                    "semantic_tags": ["inventory"],
+                }
+            ],
+        }
+
+        questions = saver.extract_questions_from_spec(spec)
+        snapshot = saver.build_snapshot_payload(
+            generation_id="gen-test",
+            title="AGENT | test",
+            questions=questions,
+            metadata={},
+        )
+
+        self.assertEqual(
+            questions,
+            ['Какие остатки на складе у ООО "Альтернатива Плюс"?'],
+        )
+        self.assertEqual(snapshot["questions"], questions)
+        self.assertEqual(snapshot["session"]["items"][0]["text"], questions[0])
+        self.assertNotIn("Рљ", questions[0])
+
     def test_extract_questions_refuses_unresolved_bindings(self) -> None:
         spec = {
             "questions": ["Что с НДС за {{bindings.control_year}} год?"],