Сохранить HM VAT old-stock AGENT pack без mojibake

llm_normalizer/data/autorun_generators/history.json

@@ -1,4 +1,62 @@
 [
+  {
+    "generation_id": "gen-ag06020454-07ff44",
+    "created_at": "2026-06-02T04:54:21+00:00",
+    "mode": "saved_user_sessions",
+    "title": "AGENT | HM VAT explicit period + old stock",
+    "count": 9,
+    "domain": "assistant_stage1_hm_vat_period_oldstock_regression",
+    "questions": [
+      "Какие остатки на складе у ООО \"Альтернатива Плюс\"?",
+      "март 2016",
+      "По выбранному объекту \"Рабочая станция универсального специалиста (индивидуальное изготовление)\": где взяли это?",
+      "а кому продали?",
+      "у тебя написано кто контрагент: рабочая станция - это ошибка?",
+      "ндс можешь прикинуть на дату покупки рабочей станции?",
+      "а какой ндс мы должны сгрузить на март 2020?",
+      "прикинь какой ндс нам надо заплатить на февраль 2017",
+      "Есть ли у ООО \"Альтернатива Плюс\" остатки товара, которые закупались очень давно"
+    ],
+    "generated_by": "codex_agent",
+    "saved_case_set_file": "assistant_autogen_saved_user_sessions_20260602045421_gen-ag06020454-07ff44.json",
+    "context": {
+      "llm_provider": null,
+      "model": null,
+      "assistant_prompt_version": null,
+      "decomposition_prompt_version": null,
+      "prompt_fingerprint": null,
+      "autogen_personality_id": null,
+      "autogen_personality_prompt": null,
+      "source_session_id": null,
+      "saved_session_file": "assistant_saved_session_20260602045421_gen-ag06020454-07ff44.json",
+      "saved_case_set_kind": "agent_semantic_scenario",
+      "agent_run": true,
+      "agent_focus": "VAT explicit period override after purchase-date bridge + factual positive old-stock aging",
+      "architecture_phase": "assistant_stage1_hm_vat_oldstock_hardening",
+      "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\assistant_stage1_hm_vat_period_oldstock_regression_20260525.json",
+      "scenario_id": "assistant_stage1_hm_vat_period_oldstock_regression_20260525",
+      "semantic_tags": [
+        "date_carryover",
+        "explicit_period_override",
+        "field_truth",
+        "historical_snapshot",
+        "inventory",
+        "item_vs_counterparty",
+        "old_stock",
+        "organization_scope",
+        "positive_stock_basis",
+        "purchase_date_bridge",
+        "purchase_provenance",
+        "sale_trace",
+        "selected_object",
+        "temporal_honesty",
+        "vat"
+      ],
+      "validation_status": "accepted_live_replay",
+      "validated_run_dir": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7",
+      "saved_after_validated_replay": true
+    }
+  },
   {
     "generation_id": "gen-ag06012117-dc9b60",
     "created_at": "2026-06-01T21:17:39+00:00",

llm_normalizer/data/autorun_generators/saved_sessions/assistant_saved_session_20260602045421_gen-ag06020454-07ff44.json

@@ -0,0 +1,225 @@
+{
+  "saved_at": "2026-06-02T04:54:21+00:00",
+  "generation_id": "gen-ag06020454-07ff44",
+  "mode": "saved_user_sessions",
+  "title": "AGENT | HM VAT explicit period + old stock",
+  "agent_run": true,
+  "questions": [
+    "Какие остатки на складе у ООО \"Альтернатива Плюс\"?",
+    "март 2016",
+    "По выбранному объекту \"Рабочая станция универсального специалиста (индивидуальное изготовление)\": где взяли это?",
+    "а кому продали?",
+    "у тебя написано кто контрагент: рабочая станция - это ошибка?",
+    "ндс можешь прикинуть на дату покупки рабочей станции?",
+    "а какой ндс мы должны сгрузить на март 2020?",
+    "прикинь какой ндс нам надо заплатить на февраль 2017",
+    "Есть ли у ООО \"Альтернатива Плюс\" остатки товара, которые закупались очень давно"
+  ],
+  "metadata": {
+    "assistant_prompt_version": null,
+    "decomposition_prompt_version": null,
+    "prompt_fingerprint": null,
+    "agent_focus": "VAT explicit period override after purchase-date bridge + factual positive old-stock aging",
+    "architecture_phase": "assistant_stage1_hm_vat_oldstock_hardening",
+    "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\assistant_stage1_hm_vat_period_oldstock_regression_20260525.json",
+    "scenario_id": "assistant_stage1_hm_vat_period_oldstock_regression_20260525",
+    "semantic_tags": [
+      "date_carryover",
+      "explicit_period_override",
+      "field_truth",
+      "historical_snapshot",
+      "inventory",
+      "item_vs_counterparty",
+      "old_stock",
+      "organization_scope",
+      "positive_stock_basis",
+      "purchase_date_bridge",
+      "purchase_provenance",
+      "sale_trace",
+      "selected_object",
+      "temporal_honesty",
+      "vat"
+    ],
+    "validation_status": "accepted_live_replay",
+    "validated_run_dir": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7",
+    "saved_after_validated_replay": true,
+    "save_gate": {
+      "schema_version": "agent_semantic_save_gate_v1",
+      "validation_status": "accepted_live_replay",
+      "validated_run_dir": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7",
+      "final_status": "accepted",
+      "review_overall_status": "pass",
+      "business_overall_status": "pass",
+      "steps_total": 9,
+      "steps_passed": 9,
+      "steps_failed": 0,
+      "steps_with_business_failures": 0,
+      "steps_with_business_warnings": 0,
+      "acceptance_gate_passed": true,
+      "effective_runtime": {
+        "manifest_path": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7\\effective_runtime.json",
+        "runner": "domain_truth_harness.run-live",
+        "git_sha": "d1ac4fc4174e337a2865566b7552a44e60e578c3",
+        "backend_url": "http://127.0.0.1:8787",
+        "mcp_proxy_url": "http://127.0.0.1:6003",
+        "llm_provider": "local",
+        "llm_model": "unsloth/qwen3-30b-a3b-instruct-2507",
+        "temperature": 0.8,
+        "max_output_tokens": 900,
+        "prompt_version": "normalizer_v2_0_2",
+        "prompt_source": "file",
+        "prompt_hash": "f36e36a5e491cd24511b380e0c0059cb01aea5e2af738fb9f01671cd291735c1",
+        "prompt_registry_status": "pass"
+      },
+      "saved_after_validated_replay": true
+    }
+  },
+  "source_session_id": null,
+  "session": {
+    "session_id": null,
+    "mode": "agent_semantic_run",
+    "items": [
+      {
+        "message_id": "agent-user-001",
+        "role": "user",
+        "text": "Какие остатки на складе у ООО \"Альтернатива Плюс\"?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-002",
+        "role": "user",
+        "text": "март 2016",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-003",
+        "role": "user",
+        "text": "По выбранному объекту \"Рабочая станция универсального специалиста (индивидуальное изготовление)\": где взяли это?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-004",
+        "role": "user",
+        "text": "а кому продали?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-005",
+        "role": "user",
+        "text": "у тебя написано кто контрагент: рабочая станция - это ошибка?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-006",
+        "role": "user",
+        "text": "ндс можешь прикинуть на дату покупки рабочей станции?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-007",
+        "role": "user",
+        "text": "а какой ндс мы должны сгрузить на март 2020?",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-008",
+        "role": "user",
+        "text": "прикинь какой ндс нам надо заплатить на февраль 2017",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      },
+      {
+        "message_id": "agent-user-009",
+        "role": "user",
+        "text": "Есть ли у ООО \"Альтернатива Плюс\" остатки товара, которые закупались очень давно",
+        "created_at": "2026-06-02T04:54:21+00:00",
+        "reply_type": null,
+        "trace_id": null,
+        "debug": null
+      }
+    ],
+    "agent_run": true,
+    "metadata": {
+      "assistant_prompt_version": null,
+      "decomposition_prompt_version": null,
+      "prompt_fingerprint": null,
+      "agent_focus": "VAT explicit period override after purchase-date bridge + factual positive old-stock aging",
+      "architecture_phase": "assistant_stage1_hm_vat_oldstock_hardening",
+      "source_spec_file": "X:\\1C\\NDC_1C\\docs\\orchestration\\assistant_stage1_hm_vat_period_oldstock_regression_20260525.json",
+      "scenario_id": "assistant_stage1_hm_vat_period_oldstock_regression_20260525",
+      "semantic_tags": [
+        "date_carryover",
+        "explicit_period_override",
+        "field_truth",
+        "historical_snapshot",
+        "inventory",
+        "item_vs_counterparty",
+        "old_stock",
+        "organization_scope",
+        "positive_stock_basis",
+        "purchase_date_bridge",
+        "purchase_provenance",
+        "sale_trace",
+        "selected_object",
+        "temporal_honesty",
+        "vat"
+      ],
+      "validation_status": "accepted_live_replay",
+      "validated_run_dir": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7",
+      "saved_after_validated_replay": true,
+      "save_gate": {
+        "schema_version": "agent_semantic_save_gate_v1",
+        "validation_status": "accepted_live_replay",
+        "validated_run_dir": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7",
+        "final_status": "accepted",
+        "review_overall_status": "pass",
+        "business_overall_status": "pass",
+        "steps_total": 9,
+        "steps_passed": 9,
+        "steps_failed": 0,
+        "steps_with_business_failures": 0,
+        "steps_with_business_warnings": 0,
+        "acceptance_gate_passed": true,
+        "effective_runtime": {
+          "manifest_path": "artifacts\\domain_runs\\hm_vat_oldstock_20260602_p7\\effective_runtime.json",
+          "runner": "domain_truth_harness.run-live",
+          "git_sha": "d1ac4fc4174e337a2865566b7552a44e60e578c3",
+          "backend_url": "http://127.0.0.1:8787",
+          "mcp_proxy_url": "http://127.0.0.1:6003",
+          "llm_provider": "local",
+          "llm_model": "unsloth/qwen3-30b-a3b-instruct-2507",
+          "temperature": 0.8,
+          "max_output_tokens": 900,
+          "prompt_version": "normalizer_v2_0_2",
+          "prompt_source": "file",
+          "prompt_hash": "f36e36a5e491cd24511b380e0c0059cb01aea5e2af738fb9f01671cd291735c1",
+          "prompt_registry_status": "pass"
+        },
+        "saved_after_validated_replay": true
+      }
+    }
+  }
+}

llm_normalizer/data/eval_cases/assistant_autogen_saved_user_sessions_20260602045421_gen-ag06020454-07ff44.json

@@ -0,0 +1,52 @@
+{
+  "suite_id": "assistant_saved_session_gen-ag06020454-07ff44",
+  "suite_version": "0.1.0",
+  "schema_version": "assistant_saved_session_suite_v0_1",
+  "generated_at": "2026-06-02T04:54:21+00:00",
+  "generation_id": "gen-ag06020454-07ff44",
+  "mode": "saved_user_sessions",
+  "title": "AGENT | HM VAT explicit period + old stock",
+  "domain": "assistant_stage1_hm_vat_period_oldstock_regression",
+  "scenario_count": 1,
+  "case_ids": [
+    "SAVED-001"
+  ],
+  "cases": [
+    {
+      "case_id": "SAVED-001",
+      "scenario_tag": "agent_saved_user_sessions",
+      "title": "AGENT | HM VAT explicit period + old stock",
+      "question_type": "followup",
+      "broadness_level": "medium",
+      "turns": [
+        {
+          "user_message": "Какие остатки на складе у ООО \"Альтернатива Плюс\"?"
+        },
+        {
+          "user_message": "март 2016"
+        },
+        {
+          "user_message": "По выбранному объекту \"Рабочая станция универсального специалиста (индивидуальное изготовление)\": где взяли это?"
+        },
+        {
+          "user_message": "а кому продали?"
+        },
+        {
+          "user_message": "у тебя написано кто контрагент: рабочая станция - это ошибка?"
+        },
+        {
+          "user_message": "ндс можешь прикинуть на дату покупки рабочей станции?"
+        },
+        {
+          "user_message": "а какой ндс мы должны сгрузить на март 2020?"
+        },
+        {
+          "user_message": "прикинь какой ндс нам надо заплатить на февраль 2017"
+        },
+        {
+          "user_message": "Есть ли у ООО \"Альтернатива Плюс\" остатки товара, которые закупались очень давно"
+        }
+      ]
+    }
+  ]
+}

scripts/save_agent_semantic_run.py

@@ -17,6 +17,7 @@ SAVED_SESSIONS_DIR = REPO_ROOT / "llm_normalizer" / "data" / "autorun_generators
 EVAL_CASES_DIR = REPO_ROOT / "llm_normalizer" / "data" / "eval_cases"
 VALIDATED_AGENT_SAVE_SCHEMA_VERSION = "agent_semantic_save_gate_v1"
 BINDING_TOKEN_RE = re.compile(r"\{\{\s*bindings\.([A-Za-z0-9_-]+)\s*\}\}")
+MOJIBAKE_MARKERS = ("Р", "С", "Ð", "Ñ")
 
 
 def now_utc() -> datetime:
@@ -38,7 +39,34 @@ def sanitize_question(value: Any) -> str:
     text = str(value or "").replace("\r\n", "\n").replace("\r", "\n")
     text = "\n".join(line.strip() for line in text.split("\n"))
     text = re.sub(r"[ \t]+", " ", text).strip()
-    return text
+    return repair_text_mojibake(text)
+
+
+def mojibake_score(value: str) -> int:
+    if not value:
+        return 0
+    score = sum(value.count(marker) for marker in MOJIBAKE_MARKERS)
+    score += value.count("\ufffd") * 5
+    score += sum(5 for char in value if "\u2500" <= char <= "\u257f")
+    return score
+
+
+def repair_text_mojibake(value: str) -> str:
+    if not value:
+        return value
+    best = value
+    best_score = mojibake_score(value)
+    for source_encoding in ("latin1", "cp1251"):
+        for target_encoding in ("utf-8", "cp1251", "cp866"):
+            try:
+                repaired = value.encode(source_encoding).decode(target_encoding)
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                continue
+            repaired_score = mojibake_score(repaired)
+            if repaired_score < best_score:
+                best = repaired
+                best_score = repaired_score
+    return best
 
 
 def normalize_bindings(raw_bindings: Any) -> dict[str, str]:
@@ -47,7 +75,7 @@ def normalize_bindings(raw_bindings: Any) -> dict[str, str]:
     result: dict[str, str] = {}
     for key, value in raw_bindings.items():
         normalized_key = str(key or "").strip()
-        normalized_value = str(value or "").strip()
+        normalized_value = repair_text_mojibake(str(value or "").strip())
         if normalized_key and normalized_value:
             result[normalized_key] = normalized_value
     return result

scripts/test_save_agent_semantic_run.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import json
 import sys
+import tempfile
 import unittest
 from pathlib import Path
-import tempfile
 
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -13,6 +13,10 @@ import agent_runtime_manifest as runtime_manifest
 import save_agent_semantic_run as saver
 
 
+def mojibake(value: str) -> str:
+    return value.encode("utf-8").decode("cp1251")
+
+
 def write_json(path: Path, payload: object) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -139,20 +143,20 @@ class SaveAgentSemanticRunTests(unittest.TestCase):
         spec = {
             "schema_version": "domain_scenario_pack_v1",
             "bindings": {
-                "main_organization": "ООО Альтернатива Плюс",
+                "main_organization": mojibake("ООО Альтернатива Плюс"),
                 "control_year": "2020",
-                "svk_counterparty": "Группа СВК",
+                "svk_counterparty": mojibake("Группа СВК"),
             },
             "scenarios": [
                 {
                     "scenario_id": "biz",
                     "steps": [
                         {
-                            "question": "Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год.",
+                            "question": mojibake("Дай обзор {{bindings.main_organization}} за {{bindings.control_year}} год."),
                             "semantic_tags": ["business_overview", "money"],
                         },
                         {
-                            "question": "Отдельно по {{bindings.svk_counterparty}} покажи документы.",
+                            "question": mojibake("Отдельно по {{bindings.svk_counterparty}} покажи документы."),
                             "semantic_tags": ["counterparty", "documents"],
                         },
                     ],
@@ -175,6 +179,36 @@ class SaveAgentSemanticRunTests(unittest.TestCase):
             ["business_overview", "counterparty", "documents", "money"],
         )
 
+    def test_extract_questions_repairs_mojibake_before_saved_session(self) -> None:
+        spec = {
+            "schema_version": "domain_truth_harness_spec_v1",
+            "bindings": {
+                "organization": mojibake('ООО "Альтернатива Плюс"'),
+            },
+            "steps": [
+                {
+                    "question": mojibake("Какие остатки на складе у {{bindings.organization}}?"),
+                    "semantic_tags": ["inventory"],
+                }
+            ],
+        }
+
+        questions = saver.extract_questions_from_spec(spec)
+        snapshot = saver.build_snapshot_payload(
+            generation_id="gen-test",
+            title="AGENT | test",
+            questions=questions,
+            metadata={},
+        )
+
+        self.assertEqual(
+            questions,
+            ['Какие остатки на складе у ООО "Альтернатива Плюс"?'],
+        )
+        self.assertEqual(snapshot["questions"], questions)
+        self.assertEqual(snapshot["session"]["items"][0]["text"], questions[0])
+        self.assertNotIn("Рљ", questions[0])
+
     def test_extract_questions_refuses_unresolved_bindings(self) -> None:
         spec = {
             "questions": ["Что с НДС за {{bindings.control_year}} год?"],