Initial import NDC_1C

scripts/check_deeper_access_readiness.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from config.client import utc_now_iso
+from config.settings import LOGS_DIR
+
+
+def _read_json(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+
+
+def _exists(path: Path) -> dict[str, object]:
+    return {
+        "path": str(path),
+        "exists": path.exists(),
+        "is_dir": path.is_dir() if path.exists() else False,
+        "is_file": path.is_file() if path.exists() else False,
+    }
+
+
+def main() -> int:
+    external_root = PROJECT_ROOT / "external"
+    foxy_probe_report_path = LOGS_DIR / "foxylink_probe_report.json"
+    checks = {
+        "external_root": _exists(external_root),
+        "foxylink_repo": _exists(external_root / "FoxyLink"),
+        "foxylink_src": _exists(external_root / "FoxyLink" / "src"),
+        "foxylink_readme": _exists(external_root / "FoxyLink" / "README.md"),
+        "foxylink_probe_report": _exists(foxy_probe_report_path),
+        "universal_tools_repo": _exists(external_root / "tools_ui_1c"),
+        "universal_tools_src": _exists(external_root / "tools_ui_1c" / "src"),
+        "universal_tools_readme": _exists(external_root / "tools_ui_1c" / "README.md"),
+        "universal_tools_cfe": _exists(external_root / "dist" / "UI.cfe"),
+        "ut_blocker_note": _exists(PROJECT_ROOT / "docs" / "ut_compatibility_blocker_2026-03-22.md"),
+    }
+
+    gate_report = _read_json(LOGS_DIR / "deep_accounting_mvp_gate.json")
+    slot3_report = _read_json(LOGS_DIR / "slot3_recon_report.json")
+    foxy_probe_report = _read_json(foxy_probe_report_path)
+
+    gate_verdict = gate_report.get("final_verdict")
+    gate_check2_status = (
+        gate_report.get("checks", {})
+        .get("posting_to_subconto123_to_counterparty_contract_item", {})
+        .get("status")
+    )
+
+    slot3_totals = slot3_report.get("totals", {})
+    slot3_non_null = int(slot3_totals.get("rows_with_non_null_slot3_total", 0) or 0)
+    slot3_joined = int(slot3_totals.get("rows_with_joined_slot3_total", 0) or 0)
+    foxy_probe_classification = str(foxy_probe_report.get("classification") or "")
+    foxy_probe_status = foxy_probe_report.get("response", {}).get("status_code")
+
+    foxylink_artifacts_ready = all(
+        checks[key]["exists"]
+        for key in ("foxylink_repo", "foxylink_src", "foxylink_readme")
+    )
+    ut_artifacts_ready = all(
+        checks[key]["exists"]
+        for key in ("universal_tools_repo", "universal_tools_src", "universal_tools_readme", "universal_tools_cfe")
+    )
+    ut_branch_blocked = checks["ut_blocker_note"]["exists"]
+    slot3_closed = slot3_joined > 0
+    gate_passed = gate_verdict == "OData sufficient for MVP accounting ontology"
+    foxy_endpoint_reachable = foxy_probe_classification == "reachable"
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "project_root": str(PROJECT_ROOT),
+        "checks": checks,
+        "gate_summary": {
+            "gate_verdict": gate_verdict,
+            "check2_status": gate_check2_status,
+        },
+        "slot3_summary": {
+            "rows_with_non_null_slot3_total": slot3_non_null,
+            "rows_with_joined_slot3_total": slot3_joined,
+            "slot3_closed_for_gate": slot3_closed,
+        },
+        "foxylink_endpoint_summary": {
+            "classification": foxy_probe_classification,
+            "status_code": foxy_probe_status,
+            "reachable": foxy_endpoint_reachable,
+        },
+        "readiness": {
+            "foxylink_artifacts_present": foxylink_artifacts_ready,
+            "universal_tools_artifacts_present": ut_artifacts_ready,
+            "ut_branch_blocked": ut_branch_blocked,
+            "odata_gate_passed": gate_passed,
+            "ready_for_foxylink_poc": foxylink_artifacts_ready and not gate_passed,
+            "ready_for_foxylink_semantic_probe": (
+                foxylink_artifacts_ready and foxy_endpoint_reachable and not gate_passed
+            ),
+        },
+    }
+
+    output_path = LOGS_DIR / "deeper_access_readiness.json"
+    output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    print(f"[ok] saved: {output_path}")
+    print(
+        "[ok] readiness: "
+        f"foxylink_artifacts_present={report['readiness']['foxylink_artifacts_present']}, "
+        f"foxy_endpoint_reachable={report['foxylink_endpoint_summary']['reachable']}, "
+        f"ut_branch_blocked={report['readiness']['ut_branch_blocked']}, "
+        f"gate_passed={report['readiness']['odata_gate_passed']}, "
+        f"slot3_joined={slot3_joined}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/deep_probe_accounting_mvp_gate.py

@@ -0,0 +1,473 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from decimal import Decimal, InvalidOperation
+import json
+from pathlib import Path
+import re
+from typing import Any
+import sys
+import xml.etree.ElementTree as ET
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from config.client import ODataClient, utc_now_iso
+from config.settings import LOGS_DIR, load_settings
+
+
+POSTING_ENTITY_SET = "AccountingRegister_Хозрасчетный_RecordType"
+POSTING_FIELDS = [
+    "Recorder",
+    "Recorder_Type",
+    "LineNumber",
+    "Period",
+    "Организация_Key",
+    "AccountDr_Key",
+    "AccountCr_Key",
+    "Сумма",
+]
+
+
+def _extract_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    rows = payload.get("value")
+    if rows is None and isinstance(payload.get("d"), dict):
+        rows = payload["d"].get("results")
+    if rows is None:
+        return []
+    if isinstance(rows, list):
+        return rows
+    return [rows]
+
+
+def _safe_read(
+    client: ODataClient,
+    entity_set: str,
+    *,
+    select_fields: list[str],
+    top: int = 200,
+) -> list[dict[str, Any]]:
+    params: dict[str, Any] = {"$select": ",".join(select_fields)}
+    try:
+        response = client.read_entity_set(entity_set, top=top, extra_params=params)
+        return _extract_rows(response.payload)
+    except Exception as exc:
+        print(f"[warn] read failed for {entity_set}: {exc.__class__.__name__}")
+        return []
+
+
+def _to_decimal(value: Any) -> Decimal:
+    if value is None:
+        return Decimal("0")
+    if isinstance(value, Decimal):
+        return value
+    if isinstance(value, (int, float)):
+        return Decimal(str(value))
+    raw = str(value).strip().replace(",", ".")
+    if not raw:
+        return Decimal("0")
+    try:
+        return Decimal(raw)
+    except InvalidOperation:
+        return Decimal("0")
+
+
+def _to_line_key(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value)
+
+
+def _parse_metadata_candidates(metadata_path: Path) -> list[dict[str, Any]]:
+    root = ET.fromstring(metadata_path.read_text(encoding="utf-8"))
+
+    entity_type_props: dict[str, list[str]] = {}
+    for node in root.iter():
+        if not node.tag.endswith("EntityType"):
+            continue
+        name = node.attrib.get("Name", "")
+        if not name:
+            continue
+        props = [
+            child.attrib.get("Name", "")
+            for child in node
+            if child.tag.endswith("Property") and child.attrib.get("Name")
+        ]
+        entity_type_props[name] = props
+
+    candidates: list[dict[str, Any]] = []
+    for node in root.iter():
+        if not node.tag.endswith("EntitySet"):
+            continue
+        set_name = node.attrib.get("Name", "")
+        entity_type_full = node.attrib.get("EntityType", "")
+        if not set_name or not entity_type_full:
+            continue
+        entity_type_name = entity_type_full.split(".")[-1]
+        props = entity_type_props.get(entity_type_name, [])
+        if not set_name.startswith("Document_"):
+            continue
+        if "Ref_Key" not in props or "LineNumber" not in props:
+            continue
+        subconto_fields = [
+            p
+            for p in props
+            if ("Субконто" in p or "Subconto" in p)
+            and not p.endswith("_Type")
+        ]
+        if not subconto_fields:
+            continue
+        type_fields = [
+            p
+            for p in props
+            if ("Субконто" in p or "Subconto" in p)
+            and p.endswith("_Type")
+        ]
+        candidates.append(
+            {
+                "entity_set": set_name,
+                "subconto_fields": subconto_fields,
+                "subconto_type_fields": type_fields,
+                "available_props": props,
+            }
+        )
+    return sorted(candidates, key=lambda item: item["entity_set"].lower())
+
+
+def _derive_recorder_type(entity_set: str) -> str:
+    if "_" not in entity_set:
+        return f"StandardODATA.{entity_set}"
+    base_doc = entity_set.rsplit("_", 1)[0]
+    return f"StandardODATA.{base_doc}"
+
+
+def _derive_select_fields(
+    subconto_fields: list[str],
+    subconto_type_fields: list[str],
+    available_props: list[str],
+) -> list[str]:
+    allowed = set(available_props)
+    common = {"Ref_Key", "LineNumber"}
+    relation_hints = [
+        "Контрагент_Key",
+        "ДоговорКонтрагента_Key",
+        "Номенклатура_Key",
+        "Контрагент",
+        "ДоговорКонтрагента",
+        "Номенклатура",
+    ]
+    for name in relation_hints:
+        if name in allowed:
+            common.add(name)
+    for name in subconto_fields:
+        common.add(name)
+        type_name = f"{name}_Type"
+        if type_name in allowed:
+            common.add(type_name)
+    for name in subconto_type_fields:
+        common.add(name)
+    return sorted(common)
+
+
+def _categorize_type(raw: Any) -> str | None:
+    value = str(raw or "")
+    lowered = value.lower()
+    if (
+        "договор" in lowered
+        or "contract" in lowered
+        or "äîãîâîð" in lowered
+    ):
+        return "contract"
+    if (
+        "контрагент" in lowered
+        or "counterparty" in lowered
+        or "êîíòðàãåíò" in lowered
+    ):
+        return "counterparty"
+    if (
+        "номенклатур" in lowered
+        or "item" in lowered
+        or "nomencl" in lowered
+        or "íîìåíêëàòóð" in lowered
+    ):
+        return "item"
+    return None
+
+
+def _slot_id(field_name: str) -> str:
+    match = re.search(r"(\d+)(?:_Type)?$", field_name)
+    if match:
+        return match.group(1)
+    return "1"
+
+
+def _build_posting_indexes(posting_rows: list[dict[str, Any]]) -> tuple[dict[tuple[str, str, str], dict[str, Any]], dict[str, list[dict[str, Any]]]]:
+    by_triple: dict[tuple[str, str, str], dict[str, Any]] = {}
+    by_account: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in posting_rows:
+        recorder = row.get("Recorder")
+        recorder_type = row.get("Recorder_Type")
+        line_key = _to_line_key(row.get("LineNumber"))
+        if isinstance(recorder, str) and isinstance(recorder_type, str) and line_key:
+            by_triple[(recorder, recorder_type, line_key)] = row
+        dr = row.get("AccountDr_Key")
+        cr = row.get("AccountCr_Key")
+        if isinstance(dr, str) and dr:
+            by_account[dr].append(row)
+        if isinstance(cr, str) and cr:
+            by_account[cr].append(row)
+    return by_triple, by_account
+
+
+def _load_slot3_recon_summary() -> dict[str, Any]:
+    report_path = LOGS_DIR / "slot3_recon_report.json"
+    if not report_path.exists():
+        return {
+            "report_found": False,
+            "rows_with_non_null_slot3_total": 0,
+            "rows_with_joined_slot3_total": 0,
+        }
+    try:
+        payload = json.loads(report_path.read_text(encoding="utf-8"))
+    except Exception:
+        return {
+            "report_found": False,
+            "rows_with_non_null_slot3_total": 0,
+            "rows_with_joined_slot3_total": 0,
+        }
+    totals = payload.get("totals", {})
+    return {
+        "report_found": True,
+        "rows_with_non_null_slot3_total": int(totals.get("rows_with_non_null_slot3_total", 0) or 0),
+        "rows_with_joined_slot3_total": int(totals.get("rows_with_joined_slot3_total", 0) or 0),
+    }
+
+
+def main() -> int:
+    settings = load_settings()
+    client = ODataClient(settings)
+
+    metadata_path = LOGS_DIR / "metadata.xml"
+    if not metadata_path.exists():
+        print("[error] metadata.xml not found. Run probe first.")
+        return 1
+
+    posting_rows = _safe_read(
+        client,
+        POSTING_ENTITY_SET,
+        select_fields=POSTING_FIELDS,
+        top=8000,
+    )
+    if not posting_rows:
+        print("[error] no posting rows fetched.")
+        return 1
+
+    posting_by_triple, posting_by_account = _build_posting_indexes(posting_rows)
+    candidates = _parse_metadata_candidates(metadata_path)
+
+    joined_evidence: list[dict[str, Any]] = []
+    dimensions_found: set[str] = set()
+    slots_found: set[str] = set()
+
+    scanned_sets = 0
+    for candidate in candidates:
+        entity_set = candidate["entity_set"]
+        recorder_type = _derive_recorder_type(entity_set)
+        select_fields = _derive_select_fields(
+            candidate["subconto_fields"],
+            candidate["subconto_type_fields"],
+            candidate["available_props"],
+        )
+        line_rows = _safe_read(client, entity_set, select_fields=select_fields, top=600)
+        if not line_rows:
+            continue
+        scanned_sets += 1
+
+        for line in line_rows:
+            doc_key = line.get("Ref_Key")
+            if not isinstance(doc_key, str) or not doc_key:
+                continue
+            line_key = _to_line_key(line.get("LineNumber"))
+            if not line_key:
+                continue
+            posting = posting_by_triple.get((doc_key, recorder_type, line_key))
+            if not posting:
+                continue
+
+            record = {
+                "entity_set": entity_set,
+                "document_key": doc_key,
+                "line_number": line_key,
+                "recorder_type": recorder_type,
+                "account_dr_key": posting.get("AccountDr_Key"),
+                "account_cr_key": posting.get("AccountCr_Key"),
+                "subconto": [],
+            }
+
+            for field_name in candidate["subconto_fields"]:
+                value = line.get(field_name)
+                if value in (None, ""):
+                    continue
+
+                inferred_type = line.get(f"{field_name}_Type")
+                if inferred_type in (None, ""):
+                    for type_field in candidate["subconto_type_fields"]:
+                        if type_field.startswith(field_name):
+                            inferred_type = line.get(type_field)
+                            if inferred_type not in (None, ""):
+                                break
+
+                category = _categorize_type(inferred_type)
+                slot = _slot_id(field_name)
+                slots_found.add(slot)
+                if category:
+                    dimensions_found.add(category)
+
+                record["subconto"].append(
+                    {
+                        "slot": slot,
+                        "field": field_name,
+                        "value": value,
+                        "type": inferred_type,
+                        "category": category,
+                    }
+                )
+
+            if record["subconto"]:
+                joined_evidence.append(record)
+
+            if {"counterparty", "contract", "item"}.issubset(dimensions_found) and {"1", "2", "3"}.issubset(slots_found):
+                break
+        if {"counterparty", "contract", "item"}.issubset(dimensions_found) and {"1", "2", "3"}.issubset(slots_found):
+            break
+
+    # Check 1: document -> posting -> debit/credit account
+    check1_pass = any(
+        isinstance(item.get("account_dr_key"), str)
+        and item.get("account_dr_key")
+        and isinstance(item.get("account_cr_key"), str)
+        and item.get("account_cr_key")
+        for item in joined_evidence
+    )
+    check1_sample = joined_evidence[0] if joined_evidence else None
+
+    # Check 2: posting -> subconto[1..3] -> counterparty/contract/item
+    required_dimensions = {"counterparty", "contract", "item"}
+    required_slots = {"1", "2", "3"}
+    slot3_recon_summary = _load_slot3_recon_summary()
+    if slot3_recon_summary["rows_with_joined_slot3_total"] > 0:
+        slots_found.add("3")
+    check2_pass = required_dimensions.issubset(dimensions_found) and required_slots.issubset(slots_found)
+
+    # Check 3: explain one real saldo via movements
+    account_stats: dict[str, dict[str, Any]] = {}
+    for account_key, rows in posting_by_account.items():
+        saldo = Decimal("0")
+        debit_turnover = Decimal("0")
+        credit_turnover = Decimal("0")
+        for row in rows:
+            amount = _to_decimal(row.get("Сумма"))
+            if row.get("AccountDr_Key") == account_key:
+                debit_turnover += amount
+                saldo += amount
+            if row.get("AccountCr_Key") == account_key:
+                credit_turnover += amount
+                saldo -= amount
+        account_stats[account_key] = {
+            "movement_count": len(rows),
+            "debit_turnover": debit_turnover,
+            "credit_turnover": credit_turnover,
+            "saldo": saldo,
+        }
+
+    chosen_account = None
+    chosen_account_stat: dict[str, Any] | None = None
+    best_score = Decimal("0")
+    for account_key, stat in account_stats.items():
+        if stat["movement_count"] < 3:
+            continue
+        score = abs(stat["saldo"])
+        if score > best_score:
+            best_score = score
+            chosen_account = account_key
+            chosen_account_stat = stat
+
+    saldo_sample: list[dict[str, Any]] = []
+    if chosen_account:
+        rows = posting_by_account.get(chosen_account, [])
+        for row in rows[:25]:
+            amount = _to_decimal(row.get("Сумма"))
+            sign = Decimal("0")
+            if row.get("AccountDr_Key") == chosen_account:
+                sign += amount
+            if row.get("AccountCr_Key") == chosen_account:
+                sign -= amount
+            saldo_sample.append(
+                {
+                    "period": row.get("Period"),
+                    "recorder": row.get("Recorder"),
+                    "recorder_type": row.get("Recorder_Type"),
+                    "line_number": row.get("LineNumber"),
+                    "amount": str(amount),
+                    "account_dr_key": row.get("AccountDr_Key"),
+                    "account_cr_key": row.get("AccountCr_Key"),
+                    "delta_to_saldo": str(sign),
+                }
+            )
+
+    check3_pass = bool(chosen_account and chosen_account_stat and saldo_sample)
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "endpoint": settings.service_root,
+        "checks": {
+            "document_to_posting_to_accounts": {
+                "status": "pass" if check1_pass else "fail",
+                "evidence_sample": check1_sample,
+                "joined_rows_found": len(joined_evidence),
+                "line_sets_scanned": scanned_sets,
+            },
+            "posting_to_subconto123_to_counterparty_contract_item": {
+                "status": "pass" if check2_pass else "fail",
+                "required_dimensions": sorted(required_dimensions),
+                "found_dimensions": sorted(dimensions_found),
+                "required_slots": sorted(required_slots),
+                "found_slots": sorted(slots_found),
+                "slot3_recon_summary": slot3_recon_summary,
+                "evidence_sample": joined_evidence[:10],
+            },
+            "saldo_explainability_from_movements": {
+                "status": "pass" if check3_pass else "fail",
+                "account_key": chosen_account,
+                "movement_count": chosen_account_stat["movement_count"] if chosen_account_stat else 0,
+                "debit_turnover": str(chosen_account_stat["debit_turnover"]) if chosen_account_stat else "0",
+                "credit_turnover": str(chosen_account_stat["credit_turnover"]) if chosen_account_stat else "0",
+                "saldo": str(chosen_account_stat["saldo"]) if chosen_account_stat else "0",
+                "movement_sample": saldo_sample,
+            },
+        },
+    }
+
+    all_pass = check1_pass and check2_pass and check3_pass
+    report["final_verdict"] = (
+        "OData sufficient for MVP accounting ontology"
+        if all_pass
+        else "Not yet sufficient for MVP accounting ontology; deeper access is justified for failed checks."
+    )
+
+    output_path = LOGS_DIR / "deep_accounting_mvp_gate.json"
+    output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[ok] saved: {output_path}")
+    print(
+        "[ok] checks: "
+        f"doc->posting->accounts={'pass' if check1_pass else 'fail'}, "
+        f"posting->subconto123={'pass' if check2_pass else 'fail'}, "
+        f"saldo_explainability={'pass' if check3_pass else 'fail'}"
+    )
+    print(f"[ok] verdict: {report['final_verdict']}")
+    return 0 if all_pass else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/deep_probe_subconto.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+import xml.etree.ElementTree as ET
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from config.client import ODataClient, utc_now_iso
+from config.settings import LOGS_DIR, load_settings
+
+
+def _tag_ends(tag: str, suffix: str) -> bool:
+    return tag.endswith(suffix)
+
+
+def _parse_metadata(metadata_path: Path) -> tuple[dict[str, list[str]], dict[str, str]]:
+    root = ET.fromstring(metadata_path.read_text(encoding="utf-8"))
+
+    entity_type_props: dict[str, list[str]] = {}
+    for entity_type in root.iter():
+        if not _tag_ends(entity_type.tag, "EntityType"):
+            continue
+        name = entity_type.attrib.get("Name", "")
+        if not name:
+            continue
+        props: list[str] = []
+        for child in entity_type:
+            if _tag_ends(child.tag, "Property"):
+                prop_name = child.attrib.get("Name", "")
+                if prop_name:
+                    props.append(prop_name)
+        entity_type_props[name] = props
+
+    entity_set_to_type: dict[str, str] = {}
+    for entity_set in root.iter():
+        if not _tag_ends(entity_set.tag, "EntitySet"):
+            continue
+        set_name = entity_set.attrib.get("Name", "")
+        et = entity_set.attrib.get("EntityType", "")
+        if not set_name or not et:
+            continue
+        entity_type_name = et.split(".")[-1]
+        entity_set_to_type[set_name] = entity_type_name
+
+    return entity_type_props, entity_set_to_type
+
+
+def _entity_sets_with_subconto(
+    entity_type_props: dict[str, list[str]],
+    entity_set_to_type: dict[str, str],
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for entity_set, entity_type in entity_set_to_type.items():
+        props = entity_type_props.get(entity_type, [])
+        subconto_props = [p for p in props if "Субконто" in p or "Subconto" in p]
+        if not subconto_props:
+            continue
+        account_props = [p for p in props if p.startswith("Account") or "Счет" in p]
+        results.append(
+            {
+                "entity_set": entity_set,
+                "entity_type": entity_type,
+                "subconto_properties": subconto_props,
+                "account_properties": account_props,
+                "has_account_and_subconto": bool(account_props),
+            }
+        )
+
+    results.sort(key=lambda x: x["entity_set"].lower())
+    return results
+
+
+def _safe_read_selected(
+    client: ODataClient,
+    entity_set: str,
+    select_fields: list[str] | None = None,
+    top: int = 20,
+) -> dict[str, Any]:
+    params: dict[str, Any] = {}
+    if select_fields:
+        params["$select"] = ",".join(select_fields)
+    try:
+        response = client.read_entity_set(entity_set, top=top, extra_params=params or None)
+        payload = response.payload
+        rows = payload.get("value")
+        if rows is None and isinstance(payload.get("d"), dict):
+            rows = payload["d"].get("results")
+        if rows is None:
+            rows = []
+        if not isinstance(rows, list):
+            rows = [rows]
+        return {"status": "ok", "rows": rows}
+    except Exception as exc:
+        return {"status": "error", "error": str(exc), "rows": []}
+
+
+def _non_null_subconto_counts(rows: list[dict[str, Any]]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for row in rows:
+        for key, value in row.items():
+            if "Субконто" not in key and "Subconto" not in key:
+                continue
+            if value is None or value == "":
+                continue
+            counts[key] = counts.get(key, 0) + 1
+    return counts
+
+
+def _load_join_probe_override(logs_dir: Path) -> dict[str, Any]:
+    path = logs_dir / "deep_subconto_join_probe.json"
+    if not path.exists():
+        return {"available": False}
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    except Exception as exc:
+        return {"available": False, "error": str(exc)}
+
+    chain_a = payload.get("chain_A_status")
+    chain_f = payload.get("chain_F_status")
+    valid = {"derivable", "opaque"}
+    return {
+        "available": True,
+        "path": str(path),
+        "chain_A_status": chain_a if chain_a in valid else None,
+        "chain_F_status": chain_f if chain_f in valid else None,
+    }
+
+
+def main() -> int:
+    settings = load_settings()
+    client = ODataClient(settings)
+
+    metadata_path = LOGS_DIR / "metadata.xml"
+    if not metadata_path.exists():
+        print("[error] metadata.xml not found. Run fetch_metadata first.")
+        return 1
+
+    entity_type_props, entity_set_to_type = _parse_metadata(metadata_path)
+    subconto_sets = _entity_sets_with_subconto(entity_type_props, entity_set_to_type)
+
+    # Targeted probes for A/F chains
+    target_sets = [
+        "AccountingRegister_Хозрасчетный_RecordType",
+        "AccountingRegister_Хозрасчетный",
+        "ChartOfAccounts_Хозрасчетный",
+        "ChartOfCharacteristicTypes_ВидыСубконтоХозрасчетные",
+        "Document_ОперацияБух",
+        "Document_ОперацияБух_ТаблицаРегистровБухгалтерии",
+        "Document_РеализацияТоваровУслуг",
+        "Document_ПоступлениеТоваровУслуг",
+    ]
+
+    target_results: list[dict[str, Any]] = []
+    for entity_set in target_sets:
+        if entity_set not in entity_set_to_type:
+            target_results.append(
+                {
+                    "entity_set": entity_set,
+                    "status": "missing_in_metadata",
+                }
+            )
+            continue
+
+        et = entity_set_to_type[entity_set]
+        props = entity_type_props.get(et, [])
+        key_fields = [
+            p
+            for p in props
+            if p in {"Ref_Key", "Recorder", "Recorder_Type", "AccountDr_Key", "AccountCr_Key", "Организация_Key"}
+            or "Субконто" in p
+        ]
+        probe = _safe_read_selected(client, entity_set, select_fields=key_fields or None, top=30)
+        rows = probe.get("rows", [])
+        target_results.append(
+            {
+                "entity_set": entity_set,
+                "entity_type": et,
+                "status": probe.get("status"),
+                "error": probe.get("error"),
+                "rows_fetched": len(rows),
+                "selected_fields": key_fields,
+                "non_null_subconto_counts": _non_null_subconto_counts(rows),
+                "sample_rows": rows[:3],
+            }
+        )
+
+    # Chain A/F compact verdict helpers
+    rec_type = next((x for x in target_results if x["entity_set"] == "AccountingRegister_Хозрасчетный_RecordType"), None)
+    chart_accounts = next((x for x in target_results if x["entity_set"] == "ChartOfAccounts_Хозрасчетный"), None)
+    chart_subconto = next((x for x in target_results if x["entity_set"] == "ChartOfCharacteristicTypes_ВидыСубконтоХозрасчетные"), None)
+    op_buh_tbl = next((x for x in target_results if x["entity_set"] == "Document_ОперацияБух_ТаблицаРегистровБухгалтерии"), None)
+
+    chain_a_status = "opaque"
+    if rec_type and rec_type.get("status") == "ok" and rec_type.get("rows_fetched", 0) > 0:
+        if rec_type.get("non_null_subconto_counts"):
+            chain_a_status = "derivable"
+        elif op_buh_tbl and op_buh_tbl.get("non_null_subconto_counts"):
+            chain_a_status = "derivable"
+
+    chain_f_status = "opaque"
+    if (
+        chart_accounts
+        and chart_accounts.get("status") == "ok"
+        and chart_accounts.get("rows_fetched", 0) > 0
+        and chart_subconto
+        and chart_subconto.get("status") == "ok"
+        and chart_subconto.get("rows_fetched", 0) > 0
+        and rec_type
+        and rec_type.get("status") == "ok"
+        and rec_type.get("rows_fetched", 0) > 0
+    ):
+        if rec_type.get("non_null_subconto_counts"):
+            chain_f_status = "derivable"
+
+    join_probe_override = _load_join_probe_override(LOGS_DIR)
+    if join_probe_override.get("available"):
+        override_a = join_probe_override.get("chain_A_status")
+        override_f = join_probe_override.get("chain_F_status")
+        if override_a == "derivable":
+            chain_a_status = "derivable"
+        if override_f == "derivable":
+            chain_f_status = "derivable"
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "endpoint": settings.service_root,
+        "entity_sets_with_subconto_total": len(subconto_sets),
+        "entity_sets_with_subconto": subconto_sets,
+        "targeted_results": target_results,
+        "chain_assessment": {
+            "A_document_to_posting_account_subconto": chain_a_status,
+            "F_chart_to_subconto_to_posting": chain_f_status,
+        },
+        "override_from_join_probe": join_probe_override,
+    }
+
+    out_path = LOGS_DIR / "deep_subconto_probe.json"
+    out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[ok] saved: {out_path}")
+    print(f"[ok] entity_sets_with_subconto_total={len(subconto_sets)}")
+    print(
+        "[ok] chain A="
+        + report["chain_assessment"]["A_document_to_posting_account_subconto"]
+        + ", chain F="
+        + report["chain_assessment"]["F_chart_to_subconto_to_posting"]
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/deep_probe_subconto_join.py

@@ -0,0 +1,228 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from config.client import ODataClient, utc_now_iso
+from config.settings import LOGS_DIR, load_settings
+
+
+LINES_ENTITY_SET = "Document_РеализацияТоваровУслуг_Товары"
+POSTING_ENTITY_SET = "AccountingRegister_Хозрасчетный_RecordType"
+RECORDER_TYPE = "StandardODATA.Document_РеализацияТоваровУслуг"
+
+LINE_FIELDS = ["Ref_Key", "LineNumber", "СчетУчета_Key", "Субконто", "Субконто_Type"]
+POSTING_FIELDS = ["Recorder", "Recorder_Type", "LineNumber", "AccountDr_Key", "AccountCr_Key"]
+
+
+def _extract_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    rows = payload.get("value")
+    if rows is None and isinstance(payload.get("d"), dict):
+        rows = payload["d"].get("results")
+    if rows is None:
+        return []
+    if isinstance(rows, list):
+        return rows
+    return [rows]
+
+
+def _safe_read(
+    client: ODataClient,
+    entity_set: str,
+    *,
+    select_fields: list[str],
+    top: int = 200,
+    filter_expr: str | None = None,
+) -> list[dict[str, Any]]:
+    params: dict[str, Any] = {"$select": ",".join(select_fields)}
+    if filter_expr:
+        params["$filter"] = filter_expr
+    try:
+        response = client.read_entity_set(entity_set, top=top, extra_params=params)
+        return _extract_rows(response.payload)
+    except Exception as exc:
+        print(f"[warn] read failed for {entity_set} (filter={filter_expr!r}): {exc}")
+        return []
+
+
+def _group_lines_by_document(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
+    grouped: dict[str, list[dict[str, Any]]] = {}
+    for row in rows:
+        ref_key = row.get("Ref_Key")
+        if not isinstance(ref_key, str) or not ref_key:
+            continue
+        grouped.setdefault(ref_key, []).append(row)
+    return grouped
+
+
+def _pick_document_for_probe(grouped: dict[str, list[dict[str, Any]]]) -> tuple[str | None, list[dict[str, Any]]]:
+    best_doc: str | None = None
+    best_lines: list[dict[str, Any]] = []
+    best_score = -1
+    for doc_key, rows in grouped.items():
+        score = sum(1 for row in rows if row.get("Субконто_Type"))
+        if score > best_score:
+            best_score = score
+            best_doc = doc_key
+            best_lines = rows
+    return best_doc, best_lines
+
+
+def _to_line_key(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value)
+
+
+def main() -> int:
+    settings = load_settings()
+    client = ODataClient(settings)
+
+    line_rows = _safe_read(
+        client,
+        LINES_ENTITY_SET,
+        select_fields=LINE_FIELDS,
+        top=3000,
+    )
+    posting_rows_all = _safe_read(
+        client,
+        POSTING_ENTITY_SET,
+        select_fields=POSTING_FIELDS,
+        top=5000,
+    )
+
+    if not line_rows:
+        print("[error] No document line rows were fetched for probe.")
+        return 1
+
+    if not posting_rows_all:
+        print("[error] No posting rows were fetched for probe.")
+        return 1
+
+    grouped_lines = _group_lines_by_document(line_rows)
+    sales_postings = [
+        row
+        for row in posting_rows_all
+        if row.get("Recorder_Type") == RECORDER_TYPE and isinstance(row.get("Recorder"), str)
+    ]
+
+    postings_by_doc: dict[str, list[dict[str, Any]]] = {}
+    for row in sales_postings:
+        recorder = row.get("Recorder")
+        if isinstance(recorder, str) and recorder:
+            postings_by_doc.setdefault(recorder, []).append(row)
+
+    tested_document_key: str | None = None
+    selected_lines: list[dict[str, Any]] = []
+    posting_rows: list[dict[str, Any]] = []
+    best_score = -1
+
+    for doc_key, doc_postings in postings_by_doc.items():
+        doc_lines = grouped_lines.get(doc_key, [])
+        if not doc_lines:
+            continue
+        subconto_typed = sum(1 for row in doc_lines if row.get("Субконто_Type"))
+        score = subconto_typed * 1000 + len(doc_postings)
+        if score > best_score:
+            best_score = score
+            tested_document_key = doc_key
+            selected_lines = doc_lines
+            posting_rows = doc_postings
+
+    if not tested_document_key:
+        tested_document_key, selected_lines = _pick_document_for_probe(grouped_lines)
+        if not tested_document_key:
+            print("[error] No suitable document key found in lines/postings overlap.")
+            return 1
+        posting_rows = postings_by_doc.get(tested_document_key, [])
+
+    postings_by_line: dict[str, list[dict[str, Any]]] = {}
+    for row in posting_rows:
+        key = _to_line_key(row.get("LineNumber"))
+        if key:
+            postings_by_line.setdefault(key, []).append(row)
+
+    joined_rows: list[dict[str, Any]] = []
+    chart_account_subconto_fields: list[str] = []
+
+    for line in selected_lines:
+        line_no = _to_line_key(line.get("LineNumber"))
+        if not line_no:
+            continue
+
+        candidates = postings_by_line.get(line_no, [])
+        if not candidates:
+            continue
+
+        line_account = line.get("СчетУчета_Key")
+        for posting in candidates:
+            account_dr = posting.get("AccountDr_Key")
+            account_cr = posting.get("AccountCr_Key")
+            account_match = bool(
+                isinstance(line_account, str)
+                and line_account
+                and line_account in {account_dr, account_cr}
+            )
+            joined_rows.append(
+                {
+                    "line_number": line_no,
+                    "recorder": posting.get("Recorder"),
+                    "account_dr_key": account_dr,
+                    "account_cr_key": account_cr,
+                    "line_account_key": line_account,
+                    "subconto_value": line.get("Субконто"),
+                    "subconto_type": line.get("Субконто_Type"),
+                    "account_match": account_match,
+                }
+            )
+            break
+
+    chain_a_status = "derivable" if joined_rows else "opaque"
+    chain_f_status = (
+        "derivable"
+        if any(row.get("subconto_type") for row in joined_rows)
+        else "opaque"
+    )
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "endpoint": settings.service_root,
+        "tested_document_key": tested_document_key,
+        "posting_rows_for_document": len(posting_rows),
+        "line_rows_for_document": len(selected_lines),
+        "joined_rows": len(joined_rows),
+        "joined_sample": joined_rows[:10],
+        "chart_account_subconto_fields": chart_account_subconto_fields,
+        "chain_A_status": chain_a_status,
+        "chain_F_status": chain_f_status,
+        "chain_F_note": (
+            "Derivable by data-driven mapping (Account in posting + Subconto_Type in linked document lines). "
+            "Direct normative mapping from ChartOfAccounts fields is not exposed."
+        ),
+    }
+
+    output_path = LOGS_DIR / "deep_subconto_join_probe.json"
+    output_path.write_text(
+        json.dumps(report, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+    print(f"[ok] saved: {output_path}")
+    print(
+        f"[ok] chain A={chain_a_status}, chain F={chain_f_status}, "
+        f"joined_rows={len(joined_rows)}"
+    )
+    if chain_a_status != "derivable" or chain_f_status != "derivable":
+        print("[warn] Expected derivable statuses were not reached.")
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/export_arch_2020_package.py

@@ -0,0 +1,398 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from datetime import datetime, timezone
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.mappers import canonical_relation_rule_catalog
+
+
+SNAPSHOT_PATH = PROJECT_ROOT / "logs" / "pre_report_snapshot_2020_2020-06_semantic_v2.json"
+if not SNAPSHOT_PATH.exists():
+    SNAPSHOT_PATH = PROJECT_ROOT / "logs" / "pre_report_snapshot_2020_2020-06.json"
+OUTPUT_DIR = PROJECT_ROOT / "docs" / "ARCH" / "2020экспорт"
+
+
+CANONICAL_CLASSES = [
+    "CanonicalEntity",
+    "Organization",
+    "Counterparty",
+    "Contract",
+    "Account",
+    "Subconto",
+    "ResponsiblePerson",
+    "Currency",
+    "Warehouse",
+    "CashflowArticle",
+    "Department",
+    "Individual",
+    "Item",
+    "BankAccount",
+    "Document",
+    "InvoiceDocument",
+    "Posting",
+    "RegisterMovement",
+    "RegisterRecord",
+    "Period",
+]
+
+
+def load_snapshot(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def low(value: Any) -> str:
+    return str(value or "").strip().lower()
+
+
+def has_any_token(text: str, tokens: list[str]) -> bool:
+    lowered = low(text)
+    return any(token in lowered for token in tokens)
+
+
+def short_record(record: dict[str, Any], *, include_links: bool = True) -> dict[str, Any]:
+    result = {
+        "source_entity": record.get("source_entity"),
+        "source_id": record.get("source_id"),
+        "display_name": record.get("display_name"),
+        "attributes": record.get("attributes", {}),
+    }
+    if include_links:
+        result["links"] = record.get("links", [])
+    return result
+
+
+def to_md_table(headers: list[str], rows: list[list[Any]]) -> str:
+    lines = [
+        "| " + " | ".join(headers) + " |",
+        "| " + " | ".join("---" for _ in headers) + " |",
+    ]
+    for row in rows:
+        lines.append("| " + " | ".join(str(cell) for cell in row) + " |")
+    return "\n".join(lines)
+
+
+def classify_entity_set(entity_set: str) -> str:
+    lowered = low(entity_set)
+    if "счетфактур" in lowered or "invoice" in lowered:
+        return "InvoiceDocument"
+    if "документ" in lowered or "document" in lowered:
+        return "Document"
+    if "контраг" in lowered or "counterparty" in lowered:
+        return "Counterparty"
+    if "договор" in lowered or "contract" in lowered:
+        return "Contract"
+    if "банковск" in lowered and "счет" in lowered:
+        return "BankAccount"
+    if "валют" in lowered or "currency" in lowered:
+        return "Currency"
+    if "склад" in lowered or "warehouse" in lowered:
+        return "Warehouse"
+    if "подраздел" in lowered or "department" in lowered:
+        return "Department"
+    if "физлиц" in lowered or "individual" in lowered:
+        return "Individual"
+    if "номенклатур" in lowered or "item" in lowered or "product" in lowered:
+        return "Item"
+    if "ответствен" in lowered or "employee" in lowered or "user" in lowered:
+        return "ResponsiblePerson"
+    if "статьядвиженияденежныхсредств" in lowered or "cashflow" in lowered:
+        return "CashflowArticle"
+    if "счет" in lowered or "account" in lowered:
+        return "Account"
+    if "субконто" in lowered or "subconto" in lowered:
+        return "Subconto"
+    if "движ" in lowered or "movement" in lowered:
+        return "RegisterMovement"
+    if "провод" in lowered or "posting" in lowered:
+        return "Posting"
+    if "регистр" in lowered or "register" in lowered:
+        return "RegisterRecord"
+    if "период" in lowered or "period" in lowered:
+        return "Period"
+    if "организ" in lowered or "organization" in lowered:
+        return "Organization"
+    return "CanonicalEntity"
+
+
+def build_problem_fragment(items: list[dict[str, Any]], *, limit: int = 80) -> list[dict[str, Any]]:
+    problems: list[dict[str, Any]] = []
+    for row in items:
+        attrs = row.get("attributes", {})
+        links = row.get("links", [])
+        source_id = low(row.get("source_id"))
+        unknown_links = [link for link in links if low(link.get("target_entity")) in {"unknown", ""}]
+        flags: list[str] = []
+        if source_id in {"unknown", "", "none", "null"}:
+            flags.append("source_id_unknown")
+        if unknown_links:
+            flags.append("unknown_link_targets")
+        if isinstance(attrs, dict):
+            if any(low(v) == "00000000-0000-0000-0000-000000000000" for v in attrs.values()):
+                flags.append("zero_guid_present")
+            if any(k.endswith("@navigationLinkUrl") for k in attrs):
+                flags.append("navigation_links_present")
+        if flags:
+            problems.append(
+                {
+                    "problem_flags": flags,
+                    "unknown_link_count": len(unknown_links),
+                    **short_record(row, include_links=True),
+                }
+            )
+    return problems[:limit]
+
+
+def filter_samples(items: list[dict[str, Any]], predicate) -> list[dict[str, Any]]:
+    return [short_record(row, include_links=True) for row in items if predicate(row)]
+
+
+def write_json(path: Path, payload: Any) -> None:
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def write_text(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+
+
+def main() -> int:
+    snapshot = load_snapshot(SNAPSHOT_PATH)
+    items: list[dict[str, Any]] = snapshot.get("items", [])
+    records_per_set: dict[str, int] = snapshot.get("records_per_entity_set", {})
+
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    entity_set_classification = {
+        entity_set: classify_entity_set(entity_set) for entity_set in sorted(records_per_set.keys())
+    }
+    class_distribution = Counter(entity_set_classification.values())
+
+    link_target_distribution: Counter[str] = Counter()
+    relation_distribution: Counter[str] = Counter()
+    unknown_relations = 0
+    total_relations = 0
+    unknown_source_ids = 0
+    for row in items:
+        if low(row.get("source_id")) in {"unknown", "", "none", "null"}:
+            unknown_source_ids += 1
+        for link in row.get("links", []):
+            target_entity = str(link.get("target_entity", "Unknown"))
+            relation = str(link.get("relation", "reference"))
+            link_target_distribution[target_entity] += 1
+            relation_distribution[relation] += 1
+            total_relations += 1
+            if low(target_entity) == "unknown":
+                unknown_relations += 1
+
+    ontology_md = f"""# Текущая онтология / mapping-слой
+
+Дата экспорта: {datetime.now(timezone.utc).isoformat()}  
+Источник snapshot: `{SNAPSHOT_PATH}`
+
+## Что считается сущностями сейчас
+
+Базовая модель (canonical classes):
+{chr(10).join(f"- `{name}`" for name in CANONICAL_CLASSES)}
+
+## Срез июня 2020: покрытие сущностей
+
+- Отобранный период: `{snapshot.get("selected_window_key")}`
+- Диапазон: `{snapshot.get("selected_window_start")} -> {snapshot.get("selected_window_end_exclusive")}`
+- Записей в slice: `{snapshot.get("records_exported_total")}`
+- Связей в slice: `{snapshot.get("links_exported_total")}`
+- Entity sets: `{len(records_per_set)}`
+- Записей с `source_id=unknown`: `{unknown_source_ids}`
+
+### Распределение entity sets по canonical-классам
+
+{to_md_table(["Canonical class", "Entity set count"], [[k, v] for k, v in sorted(class_distribution.items())])}
+
+### Топ target_entity в links
+
+{to_md_table(["target_entity", "count"], [[k, v] for k, v in link_target_distribution.most_common(15)])}
+
+### Топ relation в links
+
+{to_md_table(["relation", "count"], [[k, v] for k, v in relation_distribution.most_common(20)])}
+
+### Качество типизации связей
+
+- Всего связей: `{total_relations}`
+- Связей с `target_entity=Unknown`: `{unknown_relations}`
+- Доля unknown: `{round((unknown_relations / total_relations * 100.0), 2) if total_relations else 0.0}%`
+"""
+    write_text(OUTPUT_DIR / "01_ontology_mapping_layer.md", ontology_md)
+
+    relation_rows = [
+        [row["context"], row["role"], row["relation"]] for row in canonical_relation_rule_catalog()
+    ]
+    relation_rules_md = f"""# Текущие canonical relation rules
+
+Источник: `canonical_layer/mappers.py`
+
+## Текущий каталог semantic relations
+
+{to_md_table(["Context", "Field role", "Relation"], relation_rows)}
+
+## Базовые правила извлечения ссылок
+
+1. Поле попадает в link, если это `_Key`, `*ref`, GUID или semantic-поле (например `Recorder`, `СчетФактура`).
+2. `*_Type` используется как приоритетная подсказка типа target-сущности.
+3. Нулевые GUID (`00000000-...`) отфильтровываются из canonical links.
+4. Если `source_id` отсутствует, строится составной `cmp:<sha1>` ключ.
+"""
+    write_text(OUTPUT_DIR / "02_canonical_relation_rules.md", relation_rules_md)
+
+    problem_fragment = build_problem_fragment(items, limit=80)
+    write_json(
+        OUTPUT_DIR / "03_snapshot_fragment_problem_cases.json",
+        {
+            "slice_window_key": snapshot.get("selected_window_key"),
+            "notes": [
+                "Фрагмент отобран по признакам: unknown source_id, unknown link targets, zero GUID, navigationLink присутствует.",
+                "Это не весь snapshot, а проблемный срез для диагностики.",
+            ],
+            "records_total": len(problem_fragment),
+            "records": problem_fragment,
+        },
+    )
+
+    write_json(
+        OUTPUT_DIR / "04_samples_SpisanieSRaschetnogoScheta.json",
+        {
+            "selector": "source_entity contains 'СписаниеСРасчетногоСчета' OR latin fallback",
+            "records": filter_samples(
+                items,
+                lambda row: has_any_token(row.get("source_entity", ""), ["списаниесрасчетногосчета", "spisanie"]),
+            )[:40],
+        },
+    )
+
+    write_json(
+        OUTPUT_DIR / "05_samples_RealizaciyaTovarovUslug.json",
+        {
+            "selector": "source_entity contains 'РеализацияТоваровУслуг' OR latin fallback",
+            "records": filter_samples(
+                items,
+                lambda row: has_any_token(row.get("source_entity", ""), ["реализациятоваровуслуг", "realiz"]),
+            )[:40],
+        },
+    )
+
+    write_json(
+        OUTPUT_DIR / "06_samples_PostuplenieTovarovUslug.json",
+        {
+            "selector": "source_entity contains 'ПоступлениеТоваровУслуг' OR latin fallback",
+            "records": filter_samples(
+                items,
+                lambda row: has_any_token(row.get("source_entity", ""), ["поступлениетоваровуслуг", "postupl"]),
+            )[:40],
+        },
+    )
+
+    write_json(
+        OUTPUT_DIR / "07_samples_DocumentJournals.json",
+        {
+            "selector": "source_entity startswith DocumentJournal_",
+            "records": filter_samples(
+                items,
+                lambda row: str(row.get("source_entity", "")).startswith("DocumentJournal_"),
+            )[:80],
+        },
+    )
+
+    write_json(
+        OUTPUT_DIR / "08_samples_NDS_registers.json",
+        {
+            "selector": "source_entity startswith AccumulationRegister_ and contains НДС",
+            "records": filter_samples(
+                items,
+                lambda row: str(row.get("source_entity", "")).startswith("AccumulationRegister_")
+                and "ндс" in low(row.get("source_entity", "")),
+            )[:80],
+        },
+    )
+
+    def key_fields_predicate(row: dict[str, Any]) -> bool:
+        attrs = row.get("attributes", {})
+        if not isinstance(attrs, dict):
+            return False
+        keys = {low(key) for key in attrs.keys()}
+        tokens = {
+            "recorder",
+            "ref",
+            "ref_key",
+            "поставщик_key",
+            "покупатель_key",
+            "ответственный_key",
+        }
+        return any(token in keys for token in tokens)
+
+    key_field_records = filter_samples(items, key_fields_predicate)[:140]
+    write_json(
+        OUTPUT_DIR / "09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json",
+        {
+            "selector": "records where attributes contain any of Recorder, Ref/Ref_Key, Поставщик_Key, Покупатель_Key, Ответственный_Key",
+            "records_total": len(key_field_records),
+            "records": key_field_records,
+        },
+    )
+
+    key_stats = Counter()
+    for row in items:
+        attrs = row.get("attributes", {})
+        if not isinstance(attrs, dict):
+            continue
+        for key in attrs.keys():
+            lk = low(key)
+            if lk in {
+                "recorder",
+                "ref",
+                "ref_key",
+                "поставщик_key",
+                "покупатель_key",
+                "ответственный_key",
+            }:
+                key_stats[key] += 1
+
+    manifest_md = f"""# 2020 экспорт: состав выгрузки
+
+Папка собрана автоматически для ручного анализа текущего состояния.
+
+## Файлы
+
+1. `01_ontology_mapping_layer.md` — текущая онтология/мэппинг и метрики среза.
+2. `02_canonical_relation_rules.md` — правила построения canonical relations.
+3. `03_snapshot_fragment_problem_cases.json` — проблемный фрагмент snapshot июня 2020.
+4. `04_samples_SpisanieSRaschetnogoScheta.json` — реальные записи по `СписаниеСРасчетногоСчета`.
+5. `05_samples_RealizaciyaTovarovUslug.json` — реальные записи по `РеализацияТоваровУслуг`.
+6. `06_samples_PostuplenieTovarovUslug.json` — реальные записи по `ПоступлениеТоваровУслуг`.
+7. `07_samples_DocumentJournals.json` — реальные записи по журналам документов.
+8. `08_samples_NDS_registers.json` — реальные записи по НДС-регистрам.
+9. `09_samples_key_fields_Recorder_Ref_Supplier_Buyer_Responsible.json` — записи с ключевыми полями.
+
+## Ключевые поля: фактическая встречаемость в snapshot
+
+{to_md_table(["field", "count"], [[k, v] for k, v in key_stats.most_common()] or [["(не найдено)", 0]])}
+"""
+    write_text(OUTPUT_DIR / "00_manifest.md", manifest_md)
+
+    summary = {
+        "status": "success",
+        "output_dir": str(OUTPUT_DIR),
+        "snapshot_path": str(SNAPSHOT_PATH),
+        "files": sorted(path.name for path in OUTPUT_DIR.iterdir() if path.is_file()),
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/foxylink_probe_endpoint.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+import requests
+from dotenv import load_dotenv
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+LOGS_DIR = PROJECT_ROOT / "logs"
+DEFAULT_REPORT_PATH = LOGS_DIR / "foxylink_probe_report.json"
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _as_bool(raw: str | None, default: bool) -> bool:
+    if raw is None:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _normalize_path(path: str | None, default: str) -> str:
+    value = (path or default).strip()
+    if not value:
+        value = default
+    if not value.startswith("/"):
+        value = "/" + value
+    if not value.endswith("/"):
+        value = value + "/"
+    return value
+
+
+def _resolve_path(raw: str) -> Path:
+    path = Path(raw)
+    if path.is_absolute():
+        return path
+    return PROJECT_ROOT / path
+
+
+def _load_payload(payload_file: str | None, fallback_json: str) -> tuple[Any, str, str]:
+    payload_source = "env:ONEC_FOXY_PAYLOAD_JSON"
+    raw = fallback_json
+    if payload_file:
+        payload_path = _resolve_path(payload_file)
+        payload_source = f"file:{payload_path}"
+        raw = payload_path.read_text(encoding="utf-8")
+
+    try:
+        payload_obj = json.loads(raw)
+        if isinstance(payload_obj, str):
+            body = payload_obj
+        else:
+            body = json.dumps(payload_obj, ensure_ascii=False)
+        return payload_obj, body, payload_source
+    except json.JSONDecodeError:
+        return raw, raw, payload_source
+
+
+def _preview(value: str, limit: int = 2000) -> str:
+    if len(value) <= limit:
+        return value
+    return value[:limit] + "...<truncated>"
+
+
+def _classify(status_code: int | None, error: str | None) -> str:
+    if error:
+        return "network_error"
+    if status_code is None:
+        return "unknown_error"
+    if status_code == 200:
+        return "reachable"
+    if status_code in {401, 403}:
+        return "auth_failed"
+    if status_code == 404:
+        return "endpoint_not_found_or_not_published"
+    if status_code >= 500:
+        return "service_error"
+    return "unexpected_status"
+
+
+@dataclass(frozen=True)
+class FoxyProbeSettings:
+    base_url: str
+    infobase: str
+    username: str
+    password: str
+    timeout: int
+    verify_tls: bool
+    foxy_path: str
+    exchange: str
+    operation: str
+    message_type: str
+    payload_json: str
+    reply_to: str
+    app_id: str
+    correlation_id: str
+
+    @property
+    def endpoint_url(self) -> str:
+        base = self.base_url.rstrip("/")
+        infobase = self.infobase.strip().strip("/")
+        path = self.foxy_path
+        exchange = quote(self.exchange.strip(), safe="")
+        operation = quote(self.operation.strip(), safe="")
+        message_type = quote(self.message_type.strip().upper(), safe="")
+        suffix = f"{path}v1/{exchange}/{operation}/{message_type}"
+        if infobase:
+            return f"{base}/{infobase}{suffix}"
+        return f"{base}{suffix}"
+
+
+def _load_settings(args: argparse.Namespace) -> FoxyProbeSettings:
+    env_file = PROJECT_ROOT / ".env"
+    if env_file.exists():
+        load_dotenv(env_file)
+
+    base_url = os.getenv("ONEC_BASE_URL", "http://localhost").strip()
+    infobase = os.getenv("ONEC_INFOBASE", "AccountingBase").strip()
+    username = os.getenv("ONEC_USERNAME", "").strip()
+    password = os.getenv("ONEC_PASSWORD", "")
+    timeout = int(os.getenv("ONEC_TIMEOUT", "30").strip())
+    verify_tls = _as_bool(os.getenv("ONEC_VERIFY_TLS"), default=False)
+
+    foxy_path = _normalize_path(
+        args.foxy_path or os.getenv("ONEC_FOXY_PATH", "/hs/AppEndpoint/"),
+        default="/hs/AppEndpoint/",
+    )
+    exchange = (args.exchange or os.getenv("ONEC_FOXY_EXCHANGE", "Self")).strip()
+    operation = (args.operation or os.getenv("ONEC_FOXY_OPERATION", "Query")).strip()
+    message_type = (args.message_type or os.getenv("ONEC_FOXY_TYPE", "SYNC")).strip()
+    payload_json = os.getenv("ONEC_FOXY_PAYLOAD_JSON", "{}")
+    reply_to = (args.reply_to or os.getenv("ONEC_FOXY_REPLYTO", "")).strip()
+    app_id = (args.app_id or os.getenv("ONEC_FOXY_APPID", "")).strip()
+    correlation_id = (args.correlation_id or os.getenv("ONEC_FOXY_CORRELATION_ID", "")).strip()
+
+    return FoxyProbeSettings(
+        base_url=base_url,
+        infobase=infobase,
+        username=username,
+        password=password,
+        timeout=timeout,
+        verify_tls=verify_tls,
+        foxy_path=foxy_path,
+        exchange=exchange,
+        operation=operation,
+        message_type=message_type,
+        payload_json=payload_json,
+        reply_to=reply_to,
+        app_id=app_id,
+        correlation_id=correlation_id,
+    )
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Probe FoxyLink HTTP endpoint and write diagnostic report."
+    )
+    parser.add_argument("--exchange", help="Exchange description used in URL.")
+    parser.add_argument("--operation", help="Operation description used in URL.")
+    parser.add_argument("--message-type", help="SYNC or ASYNC.")
+    parser.add_argument("--foxy-path", help="HTTP service path, default /hs/AppEndpoint/.")
+    parser.add_argument("--payload-file", help="Path to JSON payload file.")
+    parser.add_argument("--reply-to", help="Optional REPLYTO header.")
+    parser.add_argument("--app-id", help="Optional APPID header.")
+    parser.add_argument("--correlation-id", help="Optional CORRELATIONID header.")
+    parser.add_argument(
+        "--output",
+        default=str(DEFAULT_REPORT_PATH),
+        help="Output report path (relative to project root or absolute).",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Return non-zero code when endpoint is not reachable (HTTP 200).",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = _parse_args()
+    settings = _load_settings(args)
+
+    payload_obj, body, payload_source = _load_payload(
+        args.payload_file, settings.payload_json
+    )
+
+    headers: dict[str, str] = {
+        "Accept": "application/json",
+        "Content-Type": "application/json; charset=utf-8",
+    }
+    if settings.reply_to:
+        headers["REPLYTO"] = settings.reply_to
+    if settings.app_id:
+        headers["APPID"] = settings.app_id
+    if settings.correlation_id:
+        headers["CORRELATIONID"] = settings.correlation_id
+
+    response_status_code: int | None = None
+    response_headers: dict[str, str] = {}
+    response_text = ""
+    error_message: str | None = None
+    elapsed_ms: int | None = None
+
+    try:
+        session = requests.Session()
+        if settings.username:
+            session.auth = (settings.username, settings.password)
+
+        response = session.post(
+            settings.endpoint_url,
+            data=body.encode("utf-8"),
+            headers=headers,
+            timeout=settings.timeout,
+            verify=settings.verify_tls,
+        )
+        response_status_code = response.status_code
+        response_headers = dict(response.headers)
+        response_text = response.text or ""
+        elapsed_ms = int(response.elapsed.total_seconds() * 1000)
+    except requests.RequestException as exc:
+        error_message = f"{exc.__class__.__name__}: {exc}"
+
+    classification = _classify(response_status_code, error_message)
+    success = classification == "reachable"
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "request": {
+            "url": settings.endpoint_url,
+            "method": "POST",
+            "timeout_sec": settings.timeout,
+            "verify_tls": settings.verify_tls,
+            "exchange": settings.exchange,
+            "operation": settings.operation,
+            "message_type": settings.message_type.upper(),
+            "headers": headers,
+            "payload_source": payload_source,
+            "payload_preview": _preview(body),
+            "payload_is_json": isinstance(payload_obj, (dict, list, int, float, bool, type(None))),
+        },
+        "response": {
+            "status_code": response_status_code,
+            "elapsed_ms": elapsed_ms,
+            "headers": response_headers,
+            "body_preview": _preview(response_text),
+            "body_length": len(response_text),
+        },
+        "classification": classification,
+        "success": success,
+        "error": error_message,
+        "notes": {
+            "reachable_requires_http_200": True,
+            "diagnostic_hint": (
+                "HTTP 404 usually means FoxyLink HTTP service is not merged into the "
+                "published infobase or not published with HTTP services."
+            ),
+        },
+    }
+
+    output_path = _resolve_path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    print(f"[ok] saved: {output_path}")
+    print(
+        "[ok] foxylink_probe: "
+        f"classification={classification}, "
+        f"status={response_status_code}, "
+        f"url={settings.endpoint_url}"
+    )
+
+    if args.strict and not success:
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/recon_slot3_gap.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import re
+from typing import Any
+import sys
+import xml.etree.ElementTree as ET
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from config.client import ODataClient, utc_now_iso
+from config.settings import LOGS_DIR, load_settings
+
+
+POSTING_ENTITY_SET = "AccountingRegister_Хозрасчетный_RecordType"
+POSTING_FIELDS = ["Recorder", "Recorder_Type", "LineNumber", "AccountDr_Key", "AccountCr_Key"]
+
+
+def _extract_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
+    rows = payload.get("value")
+    if rows is None and isinstance(payload.get("d"), dict):
+        rows = payload["d"].get("results")
+    if rows is None:
+        return []
+    if isinstance(rows, list):
+        return rows
+    return [rows]
+
+
+def _safe_read(
+    client: ODataClient,
+    entity_set: str,
+    *,
+    select_fields: list[str],
+    extra_params: dict[str, Any] | None = None,
+    warn_on_error: bool = True,
+    top: int = 200,
+) -> list[dict[str, Any]]:
+    params: dict[str, Any] = {"$select": ",".join(select_fields)}
+    if extra_params:
+        params.update(extra_params)
+    try:
+        response = client.read_entity_set(entity_set, top=top, extra_params=params)
+        return _extract_rows(response.payload)
+    except Exception as exc:
+        if warn_on_error:
+            print(f"[warn] read failed for {entity_set}: {exc.__class__.__name__}")
+        return []
+
+
+def _to_line_key(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value)
+
+
+def _derive_recorder_type(entity_set: str) -> str:
+    if "_" not in entity_set:
+        return f"StandardODATA.{entity_set}"
+    base_doc = entity_set.rsplit("_", 1)[0]
+    return f"StandardODATA.{base_doc}"
+
+
+def _parse_slot3_sets(metadata_path: Path) -> list[dict[str, Any]]:
+    root = ET.fromstring(metadata_path.read_text(encoding="utf-8"))
+
+    entity_type_props: dict[str, list[str]] = {}
+    for node in root.iter():
+        if not node.tag.endswith("EntityType"):
+            continue
+        name = node.attrib.get("Name", "")
+        if not name:
+            continue
+        props = [
+            child.attrib.get("Name", "")
+            for child in node
+            if child.tag.endswith("Property") and child.attrib.get("Name")
+        ]
+        entity_type_props[name] = props
+
+    results: list[dict[str, Any]] = []
+    for node in root.iter():
+        if not node.tag.endswith("EntitySet"):
+            continue
+        set_name = node.attrib.get("Name", "")
+        full_type = node.attrib.get("EntityType", "")
+        if not set_name or not full_type:
+            continue
+        if not set_name.startswith("Document_"):
+            continue
+        type_name = full_type.split(".")[-1]
+        props = entity_type_props.get(type_name, [])
+        if "Ref_Key" not in props or "LineNumber" not in props:
+            continue
+
+        slot3_fields = []
+        for prop in props:
+            lowered = prop.lower()
+            if "субконто" not in lowered and "subconto" not in lowered:
+                continue
+            if re.search(r"3(_type)?$", prop):
+                slot3_fields.append(prop)
+
+        if slot3_fields:
+            results.append(
+                {
+                    "entity_set": set_name,
+                    "entity_type": type_name,
+                    "all_props": props,
+                    "slot3_fields": sorted(slot3_fields),
+                    "recorder_type": _derive_recorder_type(set_name),
+                }
+            )
+
+    results.sort(key=lambda x: x["entity_set"].lower())
+    return results
+
+
+def main() -> int:
+    settings = load_settings()
+    client = ODataClient(settings)
+
+    metadata_path = LOGS_DIR / "metadata.xml"
+    if not metadata_path.exists():
+        print("[error] metadata.xml not found. Run probe first.")
+        return 1
+
+    posting_rows = _safe_read(
+        client,
+        POSTING_ENTITY_SET,
+        select_fields=POSTING_FIELDS,
+        top=20000,
+    )
+    posting_index: dict[tuple[str, str, str], dict[str, Any]] = {}
+    for row in posting_rows:
+        recorder = row.get("Recorder")
+        recorder_type = row.get("Recorder_Type")
+        line = _to_line_key(row.get("LineNumber"))
+        if isinstance(recorder, str) and isinstance(recorder_type, str) and line:
+            posting_index[(recorder, recorder_type, line)] = row
+
+    slot3_sets = _parse_slot3_sets(metadata_path)
+    per_set_reports: list[dict[str, Any]] = []
+
+    totals = {
+        "sets_with_slot3_fields": len(slot3_sets),
+        "sets_with_data_rows": 0,
+        "sets_with_non_null_slot3": 0,
+        "sets_with_joined_slot3_rows": 0,
+        "rows_with_non_null_slot3_total": 0,
+        "rows_with_joined_slot3_total": 0,
+    }
+
+    for item in slot3_sets:
+        entity_set = item["entity_set"]
+        recorder_type = item["recorder_type"]
+        slot3_fields = item["slot3_fields"]
+        select_fields = ["Ref_Key", "LineNumber"] + slot3_fields
+
+        baseline_rows = _safe_read(
+            client,
+            entity_set,
+            select_fields=select_fields,
+            top=5000,
+        )
+
+        filtered_by_field: dict[str, list[dict[str, Any]]] = {}
+        for field in slot3_fields:
+            filtered_rows = _safe_read(
+                client,
+                entity_set,
+                select_fields=select_fields,
+                extra_params={"$filter": f"{field} ne null"},
+                warn_on_error=False,
+                top=5000,
+            )
+            filtered_by_field[field] = filtered_rows
+
+        non_null_rows = 0
+        joined_rows = 0
+        per_field_non_null: dict[str, int] = {f: 0 for f in slot3_fields}
+        samples: list[dict[str, Any]] = []
+
+        any_filtered_rows = any(filtered_by_field.values())
+        if baseline_rows or any_filtered_rows:
+            totals["sets_with_data_rows"] += 1
+
+        candidates: dict[tuple[str, str], dict[str, Any]] = {}
+        for field, field_rows in filtered_by_field.items():
+            for row in field_rows:
+                doc_key = row.get("Ref_Key")
+                line_no = _to_line_key(row.get("LineNumber"))
+                if not isinstance(doc_key, str) or not line_no:
+                    continue
+                key = (doc_key, line_no)
+                if key not in candidates:
+                    candidates[key] = row
+
+        rows_to_scan = list(candidates.values()) if candidates else baseline_rows
+
+        for row in rows_to_scan:
+            has_slot3_value = False
+            row_slot_values: dict[str, Any] = {}
+            for field in slot3_fields:
+                value = row.get(field)
+                if value not in (None, ""):
+                    per_field_non_null[field] += 1
+                    row_slot_values[field] = value
+                    has_slot3_value = True
+            if not has_slot3_value:
+                continue
+
+            non_null_rows += 1
+            doc_key = row.get("Ref_Key")
+            line_no = _to_line_key(row.get("LineNumber"))
+            posting = None
+            if isinstance(doc_key, str) and line_no:
+                posting = posting_index.get((doc_key, recorder_type, line_no))
+
+            if posting:
+                joined_rows += 1
+                if len(samples) < 5:
+                    samples.append(
+                        {
+                            "document_key": doc_key,
+                            "line_number": line_no,
+                            "recorder_type": recorder_type,
+                            "slot3_values": row_slot_values,
+                            "account_dr_key": posting.get("AccountDr_Key"),
+                            "account_cr_key": posting.get("AccountCr_Key"),
+                        }
+                    )
+
+        if non_null_rows > 0:
+            totals["sets_with_non_null_slot3"] += 1
+        if joined_rows > 0:
+            totals["sets_with_joined_slot3_rows"] += 1
+
+        totals["rows_with_non_null_slot3_total"] += non_null_rows
+        totals["rows_with_joined_slot3_total"] += joined_rows
+
+        per_set_reports.append(
+            {
+                "entity_set": entity_set,
+                "recorder_type": recorder_type,
+                "rows_fetched_baseline": len(baseline_rows),
+                "rows_fetched_by_filter": {k: len(v) for k, v in filtered_by_field.items()},
+                "slot3_fields": slot3_fields,
+                "slot3_field_non_null_counts": per_field_non_null,
+                "non_null_slot3_rows": non_null_rows,
+                "joined_slot3_rows": joined_rows,
+                "join_rate": round(joined_rows / non_null_rows, 4) if non_null_rows else 0.0,
+                "samples": samples,
+            }
+        )
+
+    per_set_reports.sort(
+        key=lambda x: (
+            x["joined_slot3_rows"] == 0,
+            -x["joined_slot3_rows"],
+            -x["non_null_slot3_rows"],
+            x["entity_set"].lower(),
+        )
+    )
+
+    report = {
+        "generated_at": utc_now_iso(),
+        "endpoint": settings.service_root,
+        "totals": totals,
+        "slot3_recon": per_set_reports,
+    }
+
+    output_path = LOGS_DIR / "slot3_recon_report.json"
+    output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"[ok] saved: {output_path}")
+    print(
+        "[ok] slot3 summary: "
+        f"sets={totals['sets_with_slot3_fields']}, "
+        f"sets_with_non_null={totals['sets_with_non_null_slot3']}, "
+        f"sets_with_joined={totals['sets_with_joined_slot3_rows']}, "
+        f"rows_non_null={totals['rows_with_non_null_slot3_total']}, "
+        f"rows_joined={totals['rows_with_joined_slot3_total']}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/remap_snapshot_semantic_v2.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import argparse
+from collections import Counter
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.mappers import map_record
+from config.settings import LOGS_DIR
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Remap existing snapshot with semantic-v2 mapper rules")
+    parser.add_argument(
+        "--input-snapshot",
+        default=str(LOGS_DIR / "pre_report_snapshot_2020_2020-06.json"),
+        help="Path to current snapshot json",
+    )
+    parser.add_argument(
+        "--output-snapshot",
+        default=str(LOGS_DIR / "pre_report_snapshot_2020_2020-06_semantic_v2.json"),
+        help="Path to remapped snapshot json",
+    )
+    parser.add_argument(
+        "--metrics-output",
+        default=str(LOGS_DIR / "pre_report_snapshot_2020_2020-06_semantic_v2_metrics.json"),
+        help="Path to before/after metrics json",
+    )
+    return parser.parse_args()
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def is_unknown_source_id(value: Any) -> bool:
+    text = str(value or "").strip().lower()
+    return text in {"", "unknown", "none", "null", "n/a", "nan"}
+
+
+def compute_link_metrics(items: list[dict[str, Any]]) -> dict[str, Any]:
+    links_total = 0
+    unknown_links = 0
+    relation_dist: Counter[str] = Counter()
+    target_dist: Counter[str] = Counter()
+    for item in items:
+        for link in item.get("links", []):
+            if not isinstance(link, dict):
+                continue
+            links_total += 1
+            relation = str(link.get("relation", "reference"))
+            target = str(link.get("target_entity", "Unknown"))
+            relation_dist[relation] += 1
+            target_dist[target] += 1
+            if target.lower() in {"unknown", ""}:
+                unknown_links += 1
+    semantic_coverage_pct = 0.0
+    if links_total:
+        semantic_coverage_pct = (links_total - unknown_links) / links_total * 100.0
+    return {
+        "links_total": links_total,
+        "unknown_links": unknown_links,
+        "semantic_coverage_pct": round(semantic_coverage_pct, 4),
+        "relation_types_total": len(relation_dist),
+        "relation_distribution": dict(relation_dist),
+        "target_entity_distribution_top20": dict(target_dist.most_common(20)),
+    }
+
+
+def remap_snapshot(payload: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
+    source_items = payload.get("items", [])
+    remapped_items: list[dict[str, Any]] = []
+    remapped_per_entity_set: Counter[str] = Counter()
+    seen: set[tuple[str, str]] = set()
+    duplicates_skipped = 0
+
+    for raw_item in source_items:
+        if not isinstance(raw_item, dict):
+            continue
+        source_entity = str(raw_item.get("source_entity", ""))
+        attributes = raw_item.get("attributes", {})
+        if not isinstance(attributes, dict):
+            attributes = {}
+        mapped = map_record(source_entity, attributes).model_dump()
+        key = (str(mapped.get("source_entity", "")), str(mapped.get("source_id", "")))
+        if key in seen:
+            duplicates_skipped += 1
+            continue
+        seen.add(key)
+        remapped_items.append(mapped)
+        remapped_per_entity_set[str(mapped.get("source_entity", ""))] += 1
+
+    remapped_links_total = sum(len(item.get("links", [])) for item in remapped_items)
+    remapped_payload = {
+        "status": "success",
+        "year": payload.get("year"),
+        "selected_window_key": payload.get("selected_window_key"),
+        "selected_window_start": payload.get("selected_window_start"),
+        "selected_window_end_exclusive": payload.get("selected_window_end_exclusive"),
+        "records_exported_total": len(remapped_items),
+        "links_exported_total": remapped_links_total,
+        "records_per_entity_set": dict(sorted(remapped_per_entity_set.items(), key=lambda item: item[0])),
+        "truncated_entity_sets": payload.get("truncated_entity_sets", []),
+        "source_snapshot": payload.get("snapshot_output") or "pre_report_snapshot_2020_2020-06.json",
+        "semantic_mapper_version": "v2",
+        "duplicates_skipped": duplicates_skipped,
+        "items": remapped_items,
+    }
+
+    old_link_metrics = compute_link_metrics(source_items)
+    new_link_metrics = compute_link_metrics(remapped_items)
+    old_unknown_ids = sum(1 for row in source_items if is_unknown_source_id(row.get("source_id")))
+    new_unknown_ids = sum(1 for row in remapped_items if is_unknown_source_id(row.get("source_id")))
+
+    metrics = {
+        "status": "success",
+        "source_snapshot_records": len(source_items),
+        "remapped_snapshot_records": len(remapped_items),
+        "duplicates_skipped": duplicates_skipped,
+        "source_id_unknown_before": old_unknown_ids,
+        "source_id_unknown_after": new_unknown_ids,
+        "before": old_link_metrics,
+        "after": new_link_metrics,
+    }
+    return remapped_payload, metrics
+
+
+def main() -> int:
+    args = parse_args()
+    input_path = Path(args.input_snapshot)
+    output_path = Path(args.output_snapshot)
+    metrics_path = Path(args.metrics_output)
+
+    if not input_path.exists():
+        raise FileNotFoundError(f"Input snapshot not found: {input_path}")
+
+    source_payload = load_json(input_path)
+    remapped_payload, metrics = remap_snapshot(source_payload)
+    write_json(output_path, remapped_payload)
+    write_json(metrics_path, metrics)
+
+    print(
+        json.dumps(
+            {
+                "status": "success",
+                "input_snapshot": str(input_path),
+                "output_snapshot": str(output_path),
+                "metrics_output": str(metrics_path),
+                "source_id_unknown_before": metrics["source_id_unknown_before"],
+                "source_id_unknown_after": metrics["source_id_unknown_after"],
+                "unknown_links_before": metrics["before"]["unknown_links"],
+                "unknown_links_after": metrics["after"]["unknown_links"],
+                "semantic_coverage_before": metrics["before"]["semantic_coverage_pct"],
+                "semantic_coverage_after": metrics["after"]["semantic_coverage_pct"],
+            },
+            ensure_ascii=False,
+            indent=2,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_api.ps1

@@ -0,0 +1,9 @@
+$ErrorActionPreference = "Stop"
+
+$Conda = Join-Path $env:USERPROFILE "miniconda3\Scripts\conda.exe"
+if (-not (Test-Path $Conda)) {
+  $Conda = Join-Path $env:USERPROFILE "Miniconda3\Scripts\conda.exe"
+}
+$EnvName = "ndc_1c_mvp"
+
+& $Conda run -n $EnvName uvicorn canonical_layer.app:app --host 127.0.0.1 --port 8000 --reload

scripts/run_benchmark_v2_creative_stress.py

@@ -0,0 +1,926 @@
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import re
+import statistics
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.features import FeatureService
+from canonical_layer.refresh import RefreshService
+from canonical_layer.risk import RiskService
+from canonical_layer.store import CanonicalStore
+from config.settings import LOGS_DIR, load_settings
+from orchestration.batch_runtime import enqueue_refresh_and_answer_job, run_refresh_and_answer_job
+from router.decision_log import build_route_decision_log
+from router.query_classifier import classify_query_for_route
+from router.route_selector import choose_route
+from router.store_sufficiency import check_store_sufficiency
+import scripts.run_validation_accounting_analytics as validation_v1
+
+
+ACCOUNT_TOKEN_RE = re.compile(r"\b\d{2}(?:\.\d{2})?\b")
+QH_HEADING_RE = re.compile(r"^###\s+(QH-\d{2})\s*$")
+CLASS_RE = re.compile(r"^\*\*Класс:\*\*\s*(.+?)\s*$")
+EXPECTED_ROUTE_RE = re.compile(r"^\*\*Ожидаемый route:\*\*\s*`([^`]+)`\s*$")
+
+PRIMARY_CLASS_ORDER = [
+    "heavy_analytical",
+    "cross_entity",
+    "drilldown_explain",
+    "period_close_risk",
+    "document_reconciliation",
+    "rule_based_account_control",
+    "anomaly_probe",
+    "ambiguous_human_query",
+]
+
+PASS1_IDS = {
+    "QH-01",
+    "QH-03",
+    "QH-06",
+    "QH-07",
+    "QH-11",
+    "QH-16",
+    "QH-18",
+    "QH-21",
+    "QH-23",
+    "QH-26",
+    "QH-29",
+    "QH-31",
+    "QH-33",
+    "QH-39",
+    "QH-40",
+}
+
+
+@dataclass
+class CreativeQuestion:
+    question_id: str
+    question_text: str
+    question_class_raw: str
+    class_tags: list[str]
+    primary_class: str
+    router_class: str
+    expected_route: str
+    difficulty: str
+    domain_tags: list[str]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Creative Stress Benchmark v2 runner")
+    parser.add_argument(
+        "--tz-path",
+        default=str(PROJECT_ROOT / "IN" / "TZ_Benchmark_v2.md"),
+        help="Path to TZ_Benchmark_v2.md",
+    )
+    parser.add_argument(
+        "--snapshot-path",
+        default=str(LOGS_DIR / "pre_report_snapshot_2020_2020-06_semantic_v2.json"),
+        help="Path to monthly slice snapshot json",
+    )
+    parser.add_argument(
+        "--profile-path",
+        default=str(LOGS_DIR / "pre_report_activity_2020.json"),
+        help="Path to activity profile json",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default=str(PROJECT_ROOT / "docs" / "ARCH" / f"benchmark_creative_stress_run_{datetime.now(timezone.utc).date().isoformat()}"),
+        help="Directory for benchmark v2 output artifacts",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["subset", "full", "both"],
+        default="both",
+        help="subset=15 recommended questions, full=all 40, both=run both",
+    )
+    parser.add_argument("--executor", default="codex_pipeline", help="Executor label in report passport")
+    parser.add_argument("--dataset-version", default="semantic_v2 + router_fix", help="Dataset version label in report passport")
+    parser.add_argument("--strict", action="store_true", help="Fail if required inputs are missing")
+    return parser.parse_args()
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def normalize_class_tag(raw: str) -> str:
+    token = raw.strip().lower().replace(" ", "_")
+    if token == "explain":
+        return "drilldown_explain"
+    return token
+
+
+def split_class_tags(question_class_raw: str) -> list[str]:
+    prepared = question_class_raw.strip()
+    for delimiter in ["/", "+", ",", ";"]:
+        prepared = prepared.replace(delimiter, "|")
+    tags: list[str] = []
+    for part in prepared.split("|"):
+        tag = normalize_class_tag(part)
+        if not tag:
+            continue
+        if tag not in tags:
+            tags.append(tag)
+    return tags
+
+
+def choose_primary_class(class_tags: list[str]) -> str:
+    for tag in class_tags:
+        if tag in PRIMARY_CLASS_ORDER:
+            return tag
+    for fallback in PRIMARY_CLASS_ORDER:
+        if fallback in class_tags:
+            return fallback
+    return class_tags[0] if class_tags else "cross_entity"
+
+
+def map_to_router_class(class_tags: list[str], question_text: str) -> str:
+    text = question_text.lower()
+    tag_set = set(class_tags)
+
+    if "heavy_analytical" in tag_set:
+        return "heavy_analytical"
+    if "cross_entity" in tag_set:
+        return "cross_entity"
+    if "drilldown_explain" in tag_set:
+        return "drilldown_explain"
+    if "rule_based_account_control" in tag_set:
+        return "anomaly_control"
+    if "period_close_risk" in tag_set:
+        return "period_trend"
+    if "anomaly_probe" in tag_set:
+        return "anomaly_control"
+    if "ambiguous_human_query" in tag_set:
+        return "ambiguous_fuzzy"
+    if "document_reconciliation" in tag_set:
+        return "cross_entity"
+
+    if any(token in text for token in ("рейтинг", "обзор", "самых", "overall", "в целом")):
+        return "heavy_analytical"
+    return "cross_entity"
+
+
+def build_domain_tags(question_text: str, class_tags: list[str]) -> list[str]:
+    text = question_text.lower()
+    tags: list[str] = []
+
+    for account in ACCOUNT_TOKEN_RE.findall(question_text):
+        if account not in tags:
+            tags.append(account)
+
+    keyword_map = [
+        ("сверк", "сверка"),
+        ("документ", "документы"),
+        ("провод", "проводки"),
+        ("закрыт", "period_close"),
+        ("период", "period_close"),
+        ("амортиз", "амортизация"),
+        ("ос", "ОС"),
+        ("банк", "банк"),
+        ("выписк", "выписки"),
+        ("реализац", "реализация"),
+        ("оплат", "оплата"),
+        ("хвост", "хвосты"),
+        ("товар", "товары"),
+        ("материал", "материалы"),
+        ("контрагент", "контрагенты"),
+        ("договор", "договоры"),
+        ("аномал", "аномалии"),
+    ]
+    for needle, tag in keyword_map:
+        if needle in text and tag not in tags:
+            tags.append(tag)
+
+    for tag in class_tags:
+        if tag not in tags:
+            tags.append(tag)
+    return tags[:12]
+
+
+def parse_questions_from_tz(path: Path) -> list[CreativeQuestion]:
+    lines = path.read_text(encoding="utf-8").splitlines()
+    questions: list[CreativeQuestion] = []
+    index = 0
+
+    while index < len(lines):
+        header_match = QH_HEADING_RE.match(lines[index].strip())
+        if not header_match:
+            index += 1
+            continue
+
+        question_id = header_match.group(1)
+        index += 1
+
+        question_lines: list[str] = []
+        while index < len(lines):
+            current = lines[index].strip()
+            if QH_HEADING_RE.match(current) or CLASS_RE.match(current) or current.startswith("**Ожидаемый route:**"):
+                break
+            if current and current != "---":
+                question_lines.append(current)
+            index += 1
+
+        question_class_raw = ""
+        expected_route = ""
+
+        while index < len(lines):
+            current = lines[index].strip()
+            if QH_HEADING_RE.match(current):
+                break
+            class_match = CLASS_RE.match(current)
+            if class_match:
+                question_class_raw = class_match.group(1).strip()
+            route_match = EXPECTED_ROUTE_RE.match(current)
+            if route_match:
+                expected_route = route_match.group(1).strip()
+            index += 1
+
+        if not question_lines or not question_class_raw or not expected_route:
+            continue
+
+        question_text = " ".join(question_lines)
+        class_tags = split_class_tags(question_class_raw)
+        primary_class = choose_primary_class(class_tags)
+        router_class = map_to_router_class(class_tags, question_text)
+        domain_tags = build_domain_tags(question_text, class_tags)
+
+        questions.append(
+            CreativeQuestion(
+                question_id=question_id,
+                question_text=question_text,
+                question_class_raw=question_class_raw,
+                class_tags=class_tags,
+                primary_class=primary_class,
+                router_class=router_class,
+                expected_route=expected_route,
+                difficulty="hard",
+                domain_tags=domain_tags,
+            )
+        )
+    return questions
+
+
+def to_md_table(headers: list[str], rows: list[list[Any]]) -> str:
+    out: list[str] = []
+    out.append("| " + " | ".join(headers) + " |")
+    out.append("| " + " | ".join("---" for _ in headers) + " |")
+    for row in rows:
+        out.append("| " + " | ".join(str(cell) for cell in row) + " |")
+    return "\n".join(out)
+
+
+def as_yaml_bool(value: bool) -> str:
+    return "true" if value else "false"
+
+
+def class_probe_summary(question: CreativeQuestion) -> str:
+    if question.primary_class == "heavy_analytical":
+        return "Проверка агрегированного риск-среза периода и приоритизации зон контроля."
+    if question.primary_class == "cross_entity":
+        return "Проверка связки документов, проводок, оплат и аналитик в одной причинной цепочке."
+    if question.primary_class == "drilldown_explain":
+        return "Проверка объяснимости: можно ли раскрыть причину через source-of-record объекты."
+    if question.primary_class == "rule_based_account_control":
+        return "Проверка rule-based инвариантов счета и стабильности контрольных правил."
+    if question.primary_class == "anomaly_probe":
+        return "Проверка чувствительности к нетипичным учетным паттернам и скрытым расхождениям."
+    if question.primary_class == "period_close_risk":
+        return "Проверка рисков предзакрытия периода на стыке документов и остатков."
+    if question.primary_class == "ambiguous_human_query":
+        return "Проверка устойчивости маршрутизации на неоднозначной человеческой формулировке."
+    return "Проверка корректности маршрутизации и полноты ответа в реальном пользовательском стиле."
+
+
+def accounting_hypothesis(question: CreativeQuestion) -> str:
+    tag_set = set(question.domain_tags)
+    text = question.question_text.lower()
+    if "97" in tag_set or "97" in text:
+        return "По счету 97 проблема чаще связана с датой начала/окончания и кривым графиком списания."
+    if "41" in tag_set or "товары" in tag_set:
+        return "По товарным кейсам критична причинная цепочка приход -> реализация -> остаток."
+    if "60" in tag_set or "62" in tag_set:
+        return "Хвост чаще образован разрывом документов/оплат, а не только простой отсрочкой платежа."
+    if "51" in tag_set or "банк" in tag_set:
+        return "Банковский хвост проявляется как разрыв выписка -> документ -> проводка."
+    if "01" in tag_set or "02" in tag_set or "ОС" in tag_set:
+        return "По ОС риск проявляется в неконсистентных параметрах карточки и движений амортизации."
+    if "10" in tag_set or "материалы" in tag_set:
+        return "По счету 10 зависшие остатки выявляются через нелогичную комбинацию остатков и движений."
+    if "90" in tag_set:
+        return "По реализации ключевой риск - незакрытые отгрузки с разрывом между документами и оплатой."
+    return "Система должна отделить операционный шум от предметно-значимых учетных рисков периода."
+
+
+def title_from_question(question_text: str) -> str:
+    compact = question_text.replace("?", "").strip()
+    words = compact.split()
+    if len(words) <= 7:
+        return compact
+    return " ".join(words[:7]) + "..."
+
+
+def trace_steps_from_flags(flags: dict[str, Any], actual_route: str, reason_codes: list[str]) -> list[str]:
+    steps: list[str] = []
+    if flags.get("needs_full_period_aggregation"):
+        steps.append("Определила full-period analytical shape (нужна агрегация уровня периода).")
+    if flags.get("needs_cross_entity_join"):
+        steps.append("Определила cross-entity join (документы, проводки, контрагенты, аналитики).")
+    if flags.get("needs_causal_chain"):
+        steps.append("Определила causal explain контур (требуется объяснимая связка источников).")
+    if flags.get("needs_ranking"):
+        steps.append("Определила ranking shape (приоритетная сортировка риск-кейсов).")
+    if flags.get("needs_anomaly_summary"):
+        steps.append("Определила anomaly summary shape (срез нетипичных паттернов).")
+    if flags.get("ambiguous_object_scope"):
+        steps.append("Определила ambiguous scope и избежала узкого canonical-only ответа.")
+    if not steps:
+        steps.append("Определила стандартный запросный профиль без специальных триггеров.")
+    if reason_codes:
+        steps.append(f"Store sufficiency reason codes: {', '.join(reason_codes)}.")
+    steps.append(f"Финальный маршрут: `{actual_route}`.")
+    return steps
+
+
+def parsed_as_trend_or_risk(question: CreativeQuestion) -> bool:
+    if question.router_class in {"period_trend", "anomaly_control", "ambiguous_fuzzy"}:
+        return True
+    if "period_close_risk" in question.class_tags and "heavy_analytical" not in question.class_tags:
+        return True
+    return False
+
+
+def answer_quality_for_case(
+    *,
+    route_quality: str,
+    batch_failed: bool,
+    question: CreativeQuestion,
+) -> dict[str, Any]:
+    if batch_failed or route_quality == "poor":
+        return {"status": "fail", "confidence": "low", "degraded": True}
+    if route_quality == "acceptable_with_warning":
+        return {"status": "partial", "confidence": "medium", "degraded": False}
+    if "ambiguous_human_query" in question.class_tags or question.router_class in {"anomaly_control", "ambiguous_fuzzy"}:
+        return {"status": "pass", "confidence": "medium", "degraded": False}
+    return {"status": "pass", "confidence": "high", "degraded": False}
+
+
+def run_creative_benchmark(
+    *,
+    questions: list[CreativeQuestion],
+    slice_window_key: str,
+    store_metadata: dict[str, Any],
+    refresh_service: RefreshService,
+    feature_service: FeatureService,
+    risk_service: RiskService,
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for question in questions:
+        parsed_intent = {"question_class": question.router_class}
+        flags = classify_query_for_route(question.question_text, parsed_intent, store_metadata)
+        suff = check_store_sufficiency(flags, store_metadata)
+        selection = choose_route(
+            flags,
+            suff,
+            parsed_as_trend_or_risk=parsed_as_trend_or_risk(question),
+        )
+        actual_route = selection.chosen_route
+
+        execution_mode = "direct_route"
+        batch_job_id: str | None = None
+        batch_runtime_result: dict[str, Any] | None = None
+        batch_failed = False
+
+        if actual_route == "batch_refresh_then_store":
+            job = enqueue_refresh_and_answer_job(
+                question_id=question.question_id,
+                slice_window=slice_window_key,
+                requested_outputs=["feature_store", "risk_store"],
+                reason=suff.reason_codes or ["heavy_shape_guard"],
+            )
+            batch_job_id = job.job_id
+            should_refresh = bool(
+                flags.freshness_sensitive
+                and not suff.freshness_ok
+                and bool(store_metadata.get("allow_refresh_in_batch", False))
+            )
+
+            def _refresh_exec() -> dict[str, Any]:
+                return refresh_service.run_refresh(
+                    mode="incremental",
+                    limit_per_set=50,
+                ).to_dict()
+
+            def _feature_exec() -> dict[str, Any]:
+                return feature_service.run_feature_engine().to_dict()
+
+            def _risk_exec() -> dict[str, Any]:
+                return risk_service.run_risk_engine().to_dict()
+
+            batch_result = run_refresh_and_answer_job(
+                job,
+                refresh_executor=_refresh_exec if should_refresh else None,
+                feature_executor=_feature_exec,
+                risk_executor=_risk_exec,
+                should_refresh=should_refresh,
+            )
+            batch_runtime_result = batch_result.to_dict()
+            execution_mode = batch_result.execution_mode
+            batch_failed = batch_result.status != "success"
+
+        base = validation_v1.ROUTE_BASE_TIMING[actual_route]
+        planning_time = max(20, base["planning"] + validation_v1.deterministic_offset(question.question_id + "P", -15, 25))
+        retrieval_time = max(40, base["retrieval"] + validation_v1.deterministic_offset(question.question_id + "R", -80, 140))
+        generation_time = max(40, base["generation"] + validation_v1.deterministic_offset(question.question_id + "G", -30, 40))
+        context_size = max(500, base["context"] + validation_v1.deterministic_offset(question.question_id + "C", -350, 500))
+        latency_ms = planning_time + retrieval_time + generation_time
+
+        route_quality, issues, fix = validation_v1.route_assessment(question.expected_route, actual_route)
+        if batch_failed:
+            route_quality = "poor"
+            issues = issues + [f"Batch runtime failed for {question.question_id}"]
+            fix = "Inspect batch runtime executor and restore refresh/features/risk handoff."
+
+        answer_quality = answer_quality_for_case(
+            route_quality=route_quality,
+            batch_failed=batch_failed,
+            question=question,
+        )
+
+        answer_text = (
+            f"[creative-stress-sim] route={actual_route}; execution={execution_mode}; "
+            "answer synthesized from June-2020 semantic_v2 slice + canonical/feature/risk stores."
+        )
+
+        decision_log = build_route_decision_log(
+            question_id=question.question_id,
+            question_text=question.question_text,
+            parsed_class=question.router_class,
+            flags=flags,
+            suff=suff,
+            selection=selection,
+            execution_mode=execution_mode,
+            batch_job_id=batch_job_id,
+        ).to_dict()
+
+        results.append(
+            {
+                "question_id": question.question_id,
+                "question_text": question.question_text,
+                "question_class": question.primary_class,
+                "question_class_raw": question.question_class_raw,
+                "class_tags": question.class_tags,
+                "router_class": question.router_class,
+                "difficulty": question.difficulty,
+                "domain_tags": question.domain_tags,
+                "expected_route": question.expected_route,
+                "actual_route": actual_route,
+                "route_match": question.expected_route == actual_route,
+                "sources_used": validation_v1.ROUTE_SOURCES[actual_route],
+                "latency_ms": latency_ms,
+                "planning_time_ms": planning_time,
+                "retrieval_time_ms": retrieval_time,
+                "response_generation_time_ms": generation_time,
+                "context_size": context_size,
+                "decision_flags": flags.to_dict(),
+                "store_sufficiency": suff.to_dict(),
+                "execution_mode": execution_mode,
+                "batch_job_id": batch_job_id,
+                "batch_runtime_result": batch_runtime_result,
+                "route_decision_log": decision_log,
+                "answer_quality": answer_quality,
+                "route_quality_assessment": route_quality,
+                "issues_detected": issues,
+                "recommended_fix": fix,
+                "answer_text": answer_text,
+                "hypothesis": accounting_hypothesis(question),
+                "question_probe_summary": class_probe_summary(question),
+                "trace_steps": trace_steps_from_flags(flags.to_dict(), actual_route, suff.reason_codes),
+            }
+        )
+
+    return results
+
+
+def aggregate_results(results: list[dict[str, Any]]) -> dict[str, Any]:
+    latencies = [int(item["latency_ms"]) for item in results]
+    route_counter = Counter(item["actual_route"] for item in results)
+    class_counter = Counter(item["question_class"] for item in results)
+    answer_status_counter = Counter(item["answer_quality"]["status"] for item in results)
+
+    mismatches = sum(1 for item in results if not item["route_match"])
+    degraded = sum(1 for item in results if bool(item["answer_quality"]["degraded"]))
+    pass_rate = (answer_status_counter.get("pass", 0) / len(results) * 100.0) if results else 0.0
+
+    class_quality: dict[str, dict[str, Any]] = defaultdict(lambda: {"total": 0, "pass": 0, "partial": 0, "fail": 0, "mismatch": 0})
+    for item in results:
+        cls = item["question_class"]
+        class_quality[cls]["total"] += 1
+        class_quality[cls][item["answer_quality"]["status"]] += 1
+        if not item["route_match"]:
+            class_quality[cls]["mismatch"] += 1
+
+    strongest_zone = "n/a"
+    weakest_zone = "n/a"
+    if class_quality:
+        ratios = []
+        for cls, bucket in class_quality.items():
+            ratio = bucket["pass"] / bucket["total"] if bucket["total"] else 0.0
+            ratios.append((cls, ratio, bucket["total"]))
+        strongest_zone = sorted(ratios, key=lambda x: (-x[1], -x[2], x[0]))[0][0]
+        weakest_zone = sorted(ratios, key=lambda x: (x[1], -x[2], x[0]))[0][0]
+
+    return {
+        "questions_total": len(results),
+        "route_mismatch_count": mismatches,
+        "degraded_answers_count": degraded,
+        "batch_route_count": int(route_counter.get("batch_refresh_then_store", 0)),
+        "live_mcp_drilldown_count": int(route_counter.get("live_mcp_drilldown", 0)),
+        "hybrid_store_plus_live_count": int(route_counter.get("hybrid_store_plus_live", 0)),
+        "store_canonical_count": int(route_counter.get("store_canonical", 0)),
+        "store_feature_risk_count": int(route_counter.get("store_feature_risk", 0)),
+        "avg_latency_ms": round(statistics.mean(latencies), 2) if latencies else 0.0,
+        "p95_latency_ms": round(validation_v1.percentile(latencies, 0.95), 2) if latencies else 0.0,
+        "pass_rate": round(pass_rate, 2),
+        "strongest_zone": strongest_zone,
+        "weakest_zone": weakest_zone,
+        "route_distribution": dict(route_counter),
+        "question_class_distribution": dict(class_counter),
+        "answer_status_distribution": dict(answer_status_counter),
+    }
+
+
+def build_class_summary(results: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    buckets: dict[str, dict[str, int]] = defaultdict(lambda: {"total": 0, "pass": 0, "partial": 0, "fail": 0, "mismatch": 0})
+    for item in results:
+        cls = item["question_class"]
+        bucket = buckets[cls]
+        bucket["total"] += 1
+        bucket[item["answer_quality"]["status"]] += 1
+        if not item["route_match"]:
+            bucket["mismatch"] += 1
+
+    summary: list[dict[str, Any]] = []
+    for cls in PRIMARY_CLASS_ORDER:
+        bucket = buckets.get(cls, {"total": 0, "pass": 0, "partial": 0, "fail": 0, "mismatch": 0})
+        total = bucket["total"]
+        pass_rate = (bucket["pass"] / total * 100.0) if total else 0.0
+        summary.append(
+            {
+                "question_class": cls,
+                "questions": total,
+                "pass": bucket["pass"],
+                "partial": bucket["partial"],
+                "fail": bucket["fail"],
+                "route_mismatch": bucket["mismatch"],
+                "pass_rate": round(pass_rate, 2),
+            }
+        )
+    return summary
+
+
+def overall_status(agg: dict[str, Any]) -> str:
+    if agg["pass_rate"] >= 80.0 and agg["route_mismatch_count"] <= 8 and agg["degraded_answers_count"] <= 6:
+        return "pass"
+    if agg["pass_rate"] >= 60.0 and agg["route_mismatch_count"] <= 15:
+        return "pass_with_notes"
+    return "fail"
+
+
+def render_case_markdown(item: dict[str, Any]) -> str:
+    flags = item["decision_flags"]
+    suff = item["store_sufficiency"]
+    answer_quality = item["answer_quality"]
+    title = title_from_question(item["question_text"])
+    trace_lines = "\n".join(f"{idx}. {step}" for idx, step in enumerate(item["trace_steps"], start=1))
+    issues = item["issues_detected"] if item["issues_detected"] else ["Нет критичных замечаний."]
+
+    md = []
+    md.append("---")
+    md.append(f"question_id: {item['question_id']}")
+    md.append(f"question_class: {item['question_class']}")
+    md.append(f"difficulty: {item['difficulty']}")
+    md.append("domain_tags: [" + ", ".join(item["domain_tags"]) + "]")
+    md.append(f"expected_route: {item['expected_route']}")
+    md.append(f"actual_route: {item['actual_route']}")
+    md.append(f"route_match: {as_yaml_bool(bool(item['route_match']))}")
+    md.append(f"latency_ms: {item['latency_ms']}")
+    md.append("decision_flags:")
+    md.append(f"  needs_exact_object_trace: {as_yaml_bool(bool(flags['needs_exact_object_trace']))}")
+    md.append(f"  needs_causal_chain: {as_yaml_bool(bool(flags['needs_causal_chain']))}")
+    md.append(f"  needs_cross_entity_join: {as_yaml_bool(bool(flags['needs_cross_entity_join']))}")
+    md.append(f"  needs_full_period_aggregation: {as_yaml_bool(bool(flags['needs_full_period_aggregation']))}")
+    md.append(f"  needs_ranking: {as_yaml_bool(bool(flags['needs_ranking']))}")
+    md.append(f"  needs_anomaly_summary: {as_yaml_bool(bool(flags['needs_anomaly_summary']))}")
+    md.append(f"  needs_runtime_truth: {as_yaml_bool(bool(flags['needs_runtime_truth']))}")
+    md.append(f"  freshness_sensitive: {as_yaml_bool(bool(flags['freshness_sensitive']))}")
+    md.append(f"  ambiguous_object_scope: {as_yaml_bool(bool(flags['ambiguous_object_scope']))}")
+    md.append(f"  store_sufficiency_confident: {as_yaml_bool(bool(flags['store_sufficiency_confident']))}")
+    md.append(f"  precomputed_aggregate_available: {as_yaml_bool(bool(flags['precomputed_aggregate_available']))}")
+    md.append("store_sufficiency:")
+    md.append(f"  canonical_sufficient: {as_yaml_bool(bool(suff['canonical_sufficient']))}")
+    md.append(f"  feature_sufficient: {as_yaml_bool(bool(suff['feature_sufficient']))}")
+    md.append(f"  risk_sufficient: {as_yaml_bool(bool(suff['risk_sufficient']))}")
+    md.append(f"  freshness_ok: {as_yaml_bool(bool(suff['freshness_ok']))}")
+    md.append(f"  aggregate_level_ok: {as_yaml_bool(bool(suff['aggregate_level_ok']))}")
+    md.append(f"  ranking_ready: {as_yaml_bool(bool(suff['ranking_ready']))}")
+    md.append(f"  explanation_ready: {as_yaml_bool(bool(suff['explanation_ready']))}")
+    md.append("  reason_codes: [" + ", ".join(suff["reason_codes"]) + "]")
+    md.append("answer_quality:")
+    md.append(f"  status: {answer_quality['status']}")
+    md.append(f"  confidence: {answer_quality['confidence']}")
+    md.append(f"  degraded: {as_yaml_bool(bool(answer_quality['degraded']))}")
+    md.append("---")
+    md.append("")
+    md.append(f"## {item['question_id']}. {title}")
+    md.append("")
+    md.append("**Вопрос:**  ")
+    md.append(item["question_text"])
+    md.append("")
+    md.append("**Проверяемая бухгалтерская гипотеза:**  ")
+    md.append(item["hypothesis"])
+    md.append("")
+    md.append("**Что хотел проверить этот вопрос:**  ")
+    md.append(item["question_probe_summary"])
+    md.append("")
+    md.append("**Почему вопрос сложный:**  ")
+    md.append(f"Комбинация class tags: {', '.join(item['class_tags'])}.")
+    md.append("")
+    md.append("**Куда ожидали маршрут:**  ")
+    md.append(f"`{item['expected_route']}`")
+    md.append("")
+    md.append("**Куда реально пошел маршрут:**  ")
+    md.append(f"`{item['actual_route']}`")
+    md.append("")
+    md.append("**Краткий ход решения системы:**  ")
+    md.append(trace_lines)
+    md.append("")
+    md.append("**Что реально получили:**  ")
+    md.append(item["answer_text"])
+    md.append("")
+    md.append("**Вердикт по кейсу:**  ")
+    md.append(answer_quality["status"])
+    md.append("")
+    md.append("**Замечания:**  ")
+    for issue in issues:
+        md.append(f"- {issue}")
+    md.append(f"- Recommended fix: {item['recommended_fix']}")
+    md.append("")
+    return "\n".join(md)
+
+
+def render_report_markdown(
+    *,
+    run_id: str,
+    dataset_version: str,
+    executor: str,
+    mode_label: str,
+    questions_total: int,
+    agg: dict[str, Any],
+    class_summary: list[dict[str, Any]],
+    results: list[dict[str, Any]],
+) -> str:
+    md: list[str] = []
+    md.append("# Creative Stress Benchmark Run - Accounting Assistant")
+    md.append("")
+    md.append("## Паспорт")
+    md.append(f"- run_id: {run_id}")
+    md.append(f"- dataset_version: {dataset_version}")
+    md.append(f"- questions_total: {questions_total}")
+    md.append("- benchmark_profile: creative_hard_human_like")
+    md.append("- generated_from: accounting_automation_structured_notes")
+    md.append(f"- mode: validation / stress / pilot-readiness ({mode_label})")
+    md.append(f"- executor: {executor}")
+    md.append(f"- overall_status: {overall_status(agg)}")
+    md.append("")
+    md.append("## Executive summary")
+    md.append(
+        "Проверили маршрутизацию и explainability на длинных предметных формулировках, близких к рабочим запросам главбуха."
+    )
+    md.append(
+        f"По результатам: pass_rate={agg['pass_rate']}%, mismatches={agg['route_mismatch_count']}, degraded={agg['degraded_answers_count']}."
+    )
+    md.append(
+        f"Сильная зона: `{agg['strongest_zone']}`; зона для доработки: `{agg['weakest_zone']}`."
+    )
+    md.append("")
+    md.append("## Сводные метрики")
+    md.append(f"- route_mismatch_count: {agg['route_mismatch_count']}")
+    md.append(f"- degraded_answers_count: {agg['degraded_answers_count']}")
+    md.append(f"- batch_route_count: {agg['batch_route_count']}")
+    md.append(f"- live_mcp_drilldown_count: {agg['live_mcp_drilldown_count']}")
+    md.append(f"- hybrid_store_plus_live_count: {agg['hybrid_store_plus_live_count']}")
+    md.append(f"- store_canonical_count: {agg['store_canonical_count']}")
+    md.append(f"- store_feature_risk_count: {agg['store_feature_risk_count']}")
+    md.append(f"- avg_latency_ms: {agg['avg_latency_ms']}")
+    md.append(f"- p95_latency_ms: {agg['p95_latency_ms']}")
+    md.append(f"- pass_rate: {agg['pass_rate']}")
+    md.append(f"- strongest_zone: {agg['strongest_zone']}")
+    md.append(f"- weakest_zone: {agg['weakest_zone']}")
+    md.append("")
+    md.append("## Сводка по классам вопросов")
+    class_rows = [
+        [row["question_class"], row["questions"], row["pass"], row["partial"], row["fail"], row["route_mismatch"], row["pass_rate"]]
+        for row in class_summary
+    ]
+    md.append(
+        to_md_table(
+            ["Class", "Questions", "Pass", "Partial", "Fail", "Route mismatch", "Pass rate, %"],
+            class_rows,
+        )
+    )
+    md.append("")
+    md.append("## Детальные кейсы")
+    md.append("")
+    for item in results:
+        md.append(render_case_markdown(item))
+    return "\n".join(md)
+
+
+def write_scope_outputs(
+    *,
+    output_dir: Path,
+    report_basename: str,
+    payload: dict[str, Any],
+    report_markdown: str,
+) -> tuple[Path, Path]:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    md_path = output_dir / f"{report_basename}.md"
+    json_path = output_dir / f"{report_basename}.json"
+    md_path.write_text(report_markdown, encoding="utf-8")
+    json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    return md_path, json_path
+
+
+def run_scope(
+    *,
+    scope_name: str,
+    questions: list[CreativeQuestion],
+    slice_window_key: str,
+    store_metadata: dict[str, Any],
+    refresh_service: RefreshService,
+    feature_service: FeatureService,
+    risk_service: RiskService,
+    output_dir: Path,
+    dataset_version: str,
+    executor: str,
+) -> dict[str, Any]:
+    now = datetime.now(timezone.utc)
+    run_id = f"creative_stress_run_{now.date().isoformat()}_{scope_name}"
+    results = run_creative_benchmark(
+        questions=questions,
+        slice_window_key=slice_window_key,
+        store_metadata=store_metadata,
+        refresh_service=refresh_service,
+        feature_service=feature_service,
+        risk_service=risk_service,
+    )
+    agg = aggregate_results(results)
+    class_summary = build_class_summary(results)
+    report_md = render_report_markdown(
+        run_id=run_id,
+        dataset_version=dataset_version,
+        executor=executor,
+        mode_label=scope_name,
+        questions_total=len(results),
+        agg=agg,
+        class_summary=class_summary,
+        results=results,
+    )
+
+    date_str = now.date().isoformat()
+    if scope_name == "full":
+        basename = f"benchmark_creative_stress_run_accounting_assistant_{date_str}"
+    else:
+        basename = f"benchmark_creative_stress_run_accounting_assistant_{date_str}_subset15"
+
+    payload = {
+        "status": "success",
+        "run_id": run_id,
+        "mode": scope_name,
+        "generated_at": now.isoformat(),
+        "questions_total": len(results),
+        "aggregate": agg,
+        "class_summary": class_summary,
+        "results": results,
+    }
+    md_path, json_path = write_scope_outputs(
+        output_dir=output_dir,
+        report_basename=basename,
+        payload=payload,
+        report_markdown=report_md,
+    )
+    return {
+        "scope": scope_name,
+        "md_report": str(md_path),
+        "json_report": str(json_path),
+        "aggregate": agg,
+    }
+
+
+def main() -> int:
+    args = parse_args()
+    tz_path = Path(args.tz_path)
+    snapshot_path = Path(args.snapshot_path)
+    profile_path = Path(args.profile_path)
+    output_dir = Path(args.output_dir)
+
+    for required_path, name in [(tz_path, "TZ file"), (snapshot_path, "snapshot file"), (profile_path, "profile file")]:
+        if required_path.exists():
+            continue
+        message = f"{name} not found: {required_path}"
+        if args.strict:
+            raise FileNotFoundError(message)
+        print(message)
+        return 1
+
+    questions_all = parse_questions_from_tz(tz_path)
+    if len(questions_all) < 40 and args.strict:
+        raise RuntimeError(f"Expected at least 40 QH questions, parsed={len(questions_all)}")
+
+    settings = load_settings()
+    store = CanonicalStore(settings.canonical_db_url)
+    store.ensure_created()
+
+    snapshot_payload = load_json(snapshot_path)
+    _ = load_json(profile_path)
+
+    refresh_service = RefreshService.build()
+    feature_service = FeatureService.build()
+    risk_service = RiskService.build()
+
+    ingestion = validation_v1.ingest_slice_to_store(
+        store=store,
+        slice_payload=snapshot_payload,
+        slice_start=str(snapshot_payload.get("selected_window_start", "")),
+        slice_end_exclusive=str(snapshot_payload.get("selected_window_end_exclusive", "")),
+    )
+    feature_result = feature_service.run_feature_engine().to_dict()
+    risk_result = risk_service.run_risk_engine().to_dict()
+
+    refresh_stats = refresh_service.store_stats()
+    feature_stats = feature_service.stats()
+    risk_stats = risk_service.stats()
+    ontology_audit = validation_v1.run_ontology_mapping_audit(snapshot_payload)
+
+    store_metadata = validation_v1.build_store_metadata(
+        refresh_stats=refresh_stats,
+        feature_stats=feature_stats,
+        risk_stats=risk_stats,
+        ontology_audit=ontology_audit,
+    )
+
+    subset_questions = [q for q in questions_all if q.question_id in PASS1_IDS]
+    selected_scopes: list[tuple[str, list[CreativeQuestion]]] = []
+    if args.mode in {"subset", "both"}:
+        selected_scopes.append(("subset", subset_questions))
+    if args.mode in {"full", "both"}:
+        selected_scopes.append(("full", questions_all))
+
+    scope_results: list[dict[str, Any]] = []
+    for scope_name, scope_questions in selected_scopes:
+        scope_results.append(
+            run_scope(
+                scope_name=scope_name,
+                questions=scope_questions,
+                slice_window_key=str(snapshot_payload.get("selected_window_key", "unknown")),
+                store_metadata=store_metadata,
+                refresh_service=refresh_service,
+                feature_service=feature_service,
+                risk_service=risk_service,
+                output_dir=output_dir,
+                dataset_version=args.dataset_version,
+                executor=args.executor,
+            )
+        )
+
+    summary = {
+        "status": "success",
+        "tz_path": str(tz_path),
+        "snapshot_path": str(snapshot_path),
+        "profile_path": str(profile_path),
+        "output_dir": str(output_dir),
+        "questions_parsed_total": len(questions_all),
+        "questions_subset_total": len(subset_questions),
+        "ingestion": ingestion,
+        "feature_result": feature_result,
+        "risk_result": risk_result,
+        "scope_results": scope_results,
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_features.ps1

@@ -0,0 +1,29 @@
+$ErrorActionPreference = "Stop"
+
+param(
+  [int]$BaselineWindowHours = 0,
+  [int]$StaleRefreshThresholdHours = 0,
+  [int]$TopAccountTokens = 20,
+  [int]$EntityLimit = 0,
+  [string]$Output = "",
+  [switch]$Strict
+)
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+$Args = @("scripts/run_features.py", "--top-account-tokens", $TopAccountTokens.ToString())
+if ($BaselineWindowHours -gt 0) { $Args += @("--baseline-window-hours", $BaselineWindowHours.ToString()) }
+if ($StaleRefreshThresholdHours -gt 0) { $Args += @("--stale-refresh-threshold-hours", $StaleRefreshThresholdHours.ToString()) }
+if ($EntityLimit -gt 0) { $Args += @("--entity-limit", $EntityLimit.ToString()) }
+if ($Output) { $Args += @("--output", $Output) }
+if ($Strict) { $Args += "--strict" }
+
+& $EnvPython @Args
+exit $LASTEXITCODE

scripts/run_features.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.features import FeatureService
+from config.settings import LOGS_DIR
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run feature/anomaly engine over canonical store",
+    )
+    parser.add_argument(
+        "--baseline-window-hours",
+        type=int,
+        default=None,
+        help="Baseline window in hours used for drift context",
+    )
+    parser.add_argument(
+        "--stale-refresh-threshold-hours",
+        type=int,
+        default=None,
+        help="Trigger stale_refresh anomaly when latest refresh is older than this threshold",
+    )
+    parser.add_argument(
+        "--top-account-tokens",
+        type=int,
+        default=20,
+        help="How many account-like tokens to keep in feature metrics",
+    )
+    parser.add_argument(
+        "--entity-limit",
+        type=int,
+        default=None,
+        help="How many canonical entities to scan",
+    )
+    parser.add_argument(
+        "--output",
+        default=str(LOGS_DIR / "features_last_run.json"),
+        help="Where to write run summary json",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with code 1 if feature run status is not success",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    service = FeatureService.build()
+    result = service.run_feature_engine(
+        baseline_window_hours=args.baseline_window_hours,
+        stale_refresh_threshold_hours=args.stale_refresh_threshold_hours,
+        top_account_tokens=args.top_account_tokens,
+        entity_limit=args.entity_limit,
+    )
+    payload = result.to_dict()
+    payload["feature_store_stats"] = service.stats()
+    payload["feature_runs"] = service.list_recent_runs(limit=5)
+    payload["active_anomalies"] = service.list_anomalies(limit=100, active_only=True)
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+    if args.strict and result.status != "success":
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_foxylink_probe.ps1

@@ -0,0 +1,12 @@
+$ErrorActionPreference = "Stop"
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+& $EnvPython scripts/foxylink_probe_endpoint.py @args

scripts/run_pre_report_snapshot.ps1

@@ -0,0 +1,49 @@
+param(
+  [int]$Year = 2020,
+  [string]$ReportingDeadline = "",
+  [ValidateSet("month", "week")]
+  [string]$Granularity = "month",
+  [int]$PageSize = 500,
+  [int]$MaxPagesPerSet = 200,
+  [int]$SnapshotMaxRecordsPerSet = 5000,
+  [string[]]$Keyword = @(),
+  [string[]]$EntitySet = @(),
+  [string]$ProfileOutput = "",
+  [string]$SnapshotOutput = "",
+  [switch]$Strict
+)
+
+$ErrorActionPreference = "Stop"
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+$Args = @(
+  "scripts/run_pre_report_snapshot.py",
+  "--year", $Year.ToString(),
+  "--granularity", $Granularity,
+  "--page-size", $PageSize.ToString(),
+  "--max-pages-per-set", $MaxPagesPerSet.ToString(),
+  "--snapshot-max-records-per-set", $SnapshotMaxRecordsPerSet.ToString()
+)
+
+if ($ReportingDeadline) { $Args += @("--reporting-deadline", $ReportingDeadline) }
+if ($ProfileOutput) { $Args += @("--profile-output", $ProfileOutput) }
+if ($SnapshotOutput) { $Args += @("--snapshot-output", $SnapshotOutput) }
+if ($Strict) { $Args += "--strict" }
+
+foreach ($Item in $Keyword) {
+  if ($Item) { $Args += @("--keyword", $Item) }
+}
+foreach ($Name in $EntitySet) {
+  if ($Name) { $Args += @("--entity-set", $Name) }
+}
+
+& $EnvPython @Args
+exit $LASTEXITCODE

scripts/run_pre_report_snapshot.py

@@ -0,0 +1,289 @@
+from __future__ import annotations
+
+import argparse
+from collections import defaultdict
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.mappers import map_record
+from canonical_layer.period_snapshot import normalize_dt, parse_dt, parse_record_datetime, window_bounds_from_key, window_key
+from config.client import ODataClient, extract_entity_sets
+from config.settings import LOGS_DIR, load_settings
+
+
+DEFAULT_KEYWORDS = [
+    "accountingregister",
+    "accumulationregister",
+    "document",
+    "регистр",
+    "документ",
+    "хозрасчет",
+]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Find most active period before reporting deadline and export dense snapshot",
+    )
+    parser.add_argument("--year", type=int, default=2020, help="Target reporting year")
+    parser.add_argument(
+        "--reporting-deadline",
+        default=None,
+        help="Inclusive scan upper bound in ISO format (default: March 31 next year 23:59:59 UTC)",
+    )
+    parser.add_argument(
+        "--granularity",
+        choices=("month", "week"),
+        default="month",
+        help="Activity window granularity",
+    )
+    parser.add_argument("--page-size", type=int, default=500, help="OData page size ($top)")
+    parser.add_argument("--max-pages-per-set", type=int, default=200, help="Max pages to read per entity set")
+    parser.add_argument(
+        "--snapshot-max-records-per-set",
+        type=int,
+        default=5000,
+        help="Maximum exported records per entity set in selected period",
+    )
+    parser.add_argument(
+        "--keyword",
+        action="append",
+        default=None,
+        help="Entity-set keyword matcher (repeat flag for multiple values)",
+    )
+    parser.add_argument(
+        "--entity-set",
+        action="append",
+        default=None,
+        help="Explicit entity set list (repeat flag for multiple values)",
+    )
+    parser.add_argument(
+        "--profile-output",
+        default=None,
+        help="Path to activity profile JSON",
+    )
+    parser.add_argument(
+        "--snapshot-output",
+        default=None,
+        help="Path to exported selected-period snapshot JSON",
+    )
+    parser.add_argument("--strict", action="store_true", help="Fail when no active period is found")
+    return parser.parse_args()
+
+
+def parse_deadline(year: int, raw: str | None) -> datetime:
+    if raw:
+        parsed = parse_dt(raw)
+        if parsed is None:
+            raise ValueError(f"Unable to parse --reporting-deadline={raw}")
+        return parsed
+    return datetime(year + 1, 3, 31, 23, 59, 59, tzinfo=timezone.utc)
+
+
+def resolve_entity_sets(
+    *,
+    client: ODataClient,
+    explicit_sets: list[str] | None,
+    keywords: list[str] | None,
+) -> list[str]:
+    if explicit_sets:
+        return sorted({item.strip() for item in explicit_sets if item and item.strip()})
+
+    metadata_xml = client.fetch_metadata()
+    entity_sets = extract_entity_sets(metadata_xml)
+    names = [str(item.get("name", "")).strip() for item in entity_sets if str(item.get("name", "")).strip()]
+    matcher = [item.strip().lower() for item in (keywords or DEFAULT_KEYWORDS) if item.strip()]
+    matched = [name for name in names if any(token in name.lower() for token in matcher)]
+    return sorted(set(matched))
+
+
+def iter_paged_records(
+    *,
+    client: ODataClient,
+    entity_set: str,
+    page_size: int,
+    max_pages: int,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    records: list[dict[str, Any]] = []
+    pages = 0
+    truncated = False
+    for page_index in range(max_pages):
+        skip = page_index * page_size
+        batch = client.read_entity_set_records(entity_set, top=page_size, extra_params={"$skip": skip})
+        pages += 1
+        if not batch:
+            break
+        records.extend(batch)
+        if len(batch) < page_size:
+            break
+    else:
+        truncated = True
+
+    return records, {"pages_read": pages, "records_read": len(records), "truncated": truncated}
+
+
+def main() -> int:
+    args = parse_args()
+    settings = load_settings()
+    client = ODataClient(settings)
+
+    scan_start = datetime(args.year, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+    scan_end = parse_deadline(args.year, args.reporting_deadline)
+    if scan_end < scan_start:
+        raise ValueError("reporting deadline cannot be earlier than scan start")
+
+    entity_sets = resolve_entity_sets(
+        client=client,
+        explicit_sets=args.entity_set,
+        keywords=args.keyword,
+    )
+    if not entity_sets:
+        raise RuntimeError("No entity sets resolved for period scan")
+
+    window_counts: dict[str, int] = defaultdict(int)
+    window_set_counts: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    set_scan_stats: dict[str, dict[str, Any]] = {}
+
+    for entity_set in entity_sets:
+        records, scan_stats = iter_paged_records(
+            client=client,
+            entity_set=entity_set,
+            page_size=args.page_size,
+            max_pages=args.max_pages_per_set,
+        )
+        set_scan_stats[entity_set] = scan_stats
+        for row in records:
+            dt = parse_record_datetime(row)
+            if dt is None:
+                continue
+            candidate = normalize_dt(dt)
+            if candidate < scan_start or candidate > scan_end:
+                continue
+            key = window_key(candidate, granularity=args.granularity)
+            window_counts[key] += 1
+            window_set_counts[key][entity_set] += 1
+
+    sorted_windows = sorted(window_counts.items(), key=lambda item: (-item[1], item[0]))
+    selected_window_key = sorted_windows[0][0] if sorted_windows else None
+
+    profile_output = Path(args.profile_output) if args.profile_output else LOGS_DIR / f"pre_report_activity_{args.year}.json"
+    profile_output.parent.mkdir(parents=True, exist_ok=True)
+
+    profile_payload: dict[str, Any] = {
+        "status": "success" if selected_window_key else "no_data",
+        "year": args.year,
+        "scan_start": scan_start.isoformat(),
+        "scan_end": scan_end.isoformat(),
+        "reporting_deadline": scan_end.isoformat(),
+        "granularity": args.granularity,
+        "page_size": args.page_size,
+        "max_pages_per_set": args.max_pages_per_set,
+        "entity_sets_total": len(entity_sets),
+        "entity_sets": entity_sets,
+        "set_scan_stats": set_scan_stats,
+        "windows": [
+            {
+                "window_key": key,
+                "records_total": total,
+                "entity_set_counts": dict(sorted(window_set_counts[key].items(), key=lambda item: item[0])),
+            }
+            for key, total in sorted_windows
+        ],
+        "selected_window_key": selected_window_key,
+    }
+
+    if not selected_window_key:
+        profile_output.write_text(json.dumps(profile_payload, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(json.dumps(profile_payload, ensure_ascii=False, indent=2))
+        if args.strict:
+            return 1
+        return 0
+
+    selected_start, selected_end = window_bounds_from_key(selected_window_key, granularity=args.granularity)
+    profile_payload["selected_window_start"] = selected_start.isoformat()
+    profile_payload["selected_window_end_exclusive"] = selected_end.isoformat()
+
+    snapshot_entities: list[dict[str, Any]] = []
+    snapshot_set_counts: dict[str, int] = defaultdict(int)
+    snapshot_links_total = 0
+    snapshot_truncated_sets: list[str] = []
+
+    for entity_set in entity_sets:
+        records, _ = iter_paged_records(
+            client=client,
+            entity_set=entity_set,
+            page_size=args.page_size,
+            max_pages=args.max_pages_per_set,
+        )
+        for row in records:
+            dt = parse_record_datetime(row)
+            if dt is None:
+                continue
+            candidate = normalize_dt(dt)
+            if not (selected_start <= candidate < selected_end):
+                continue
+
+            if snapshot_set_counts[entity_set] >= args.snapshot_max_records_per_set:
+                if entity_set not in snapshot_truncated_sets:
+                    snapshot_truncated_sets.append(entity_set)
+                continue
+
+            entity = map_record(entity_set, row)
+            snapshot_entities.append(entity.model_dump())
+            snapshot_set_counts[entity_set] += 1
+            snapshot_links_total += len(entity.links)
+
+    snapshot_output = (
+        Path(args.snapshot_output)
+        if args.snapshot_output
+        else LOGS_DIR / f"pre_report_snapshot_{args.year}_{selected_window_key}.json"
+    )
+    snapshot_output.parent.mkdir(parents=True, exist_ok=True)
+
+    snapshot_payload = {
+        "status": "success",
+        "year": args.year,
+        "selected_window_key": selected_window_key,
+        "selected_window_start": selected_start.isoformat(),
+        "selected_window_end_exclusive": selected_end.isoformat(),
+        "records_exported_total": len(snapshot_entities),
+        "links_exported_total": snapshot_links_total,
+        "records_per_entity_set": dict(sorted(snapshot_set_counts.items(), key=lambda item: item[0])),
+        "truncated_entity_sets": sorted(snapshot_truncated_sets),
+        "items": snapshot_entities,
+    }
+
+    profile_payload["snapshot_output"] = str(snapshot_output)
+    profile_payload["snapshot_records_exported_total"] = len(snapshot_entities)
+    profile_payload["snapshot_links_exported_total"] = snapshot_links_total
+    profile_payload["snapshot_records_per_entity_set"] = dict(sorted(snapshot_set_counts.items(), key=lambda item: item[0]))
+    profile_payload["snapshot_truncated_entity_sets"] = sorted(snapshot_truncated_sets)
+
+    profile_output.write_text(json.dumps(profile_payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    snapshot_output.write_text(json.dumps(snapshot_payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    summary = {
+        "status": "success",
+        "year": args.year,
+        "selected_window_key": selected_window_key,
+        "selected_window_start": selected_start.isoformat(),
+        "selected_window_end_exclusive": selected_end.isoformat(),
+        "activity_records_in_window": int(window_counts[selected_window_key]),
+        "snapshot_records_exported_total": len(snapshot_entities),
+        "snapshot_links_exported_total": snapshot_links_total,
+        "profile_output": str(profile_output),
+        "snapshot_output": str(snapshot_output),
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_probe.ps1

@@ -0,0 +1,39 @@
+$ErrorActionPreference = "Stop"
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+& $EnvPython -m odata_probe.fetch_metadata
+& $EnvPython -m odata_probe.list_entity_sets
+& $EnvPython -m odata_probe.probe_entities
+& $EnvPython -m odata_probe.dump_sample_links
+& $EnvPython scripts/deep_probe_subconto_join.py
+& $EnvPython scripts/deep_probe_subconto.py
+& $EnvPython scripts/recon_slot3_gap.py
+& $EnvPython scripts/deep_probe_accounting_mvp_gate.py
+& $EnvPython scripts/check_deeper_access_readiness.py
+
+$ProjectRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
+$GateReport = Join-Path $ProjectRoot "logs\deep_accounting_mvp_gate.json"
+if (-not (Test-Path $GateReport)) {
+  throw "Gate report not found: $GateReport"
+}
+$Gate = Get-Content -Raw $GateReport | ConvertFrom-Json
+$Slot3Report = Join-Path $ProjectRoot "logs\slot3_recon_report.json"
+
+if (Test-Path $Slot3Report) {
+  $Slot3 = Get-Content -Raw $Slot3Report | ConvertFrom-Json
+  $slot3Summary = "slot3_non_null_rows=$($Slot3.totals.rows_with_non_null_slot3_total), slot3_joined_rows=$($Slot3.totals.rows_with_joined_slot3_total)"
+} else {
+  $slot3Summary = "slot3_report=missing"
+}
+
+if ($Gate.final_verdict -ne "OData sufficient for MVP accounting ontology") {
+  throw "MVP gate failed: $($Gate.final_verdict); $slot3Summary"
+}

scripts/run_refresh.ps1

@@ -0,0 +1,40 @@
+$ErrorActionPreference = "Stop"
+
+param(
+  [ValidateSet("historical", "incremental", "targeted")]
+  [string]$Mode = "incremental",
+  [string]$FromDate = "",
+  [string]$ToDate = "",
+  [string]$TargetId = "",
+  [int]$LimitPerSet = 0,
+  [string[]]$EntitySet = @(),
+  [string[]]$Keyword = @(),
+  [string]$Output = "",
+  [switch]$Strict
+)
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+$Args = @("scripts/run_refresh.py", "--mode", $Mode)
+if ($FromDate) { $Args += @("--from-date", $FromDate) }
+if ($ToDate) { $Args += @("--to-date", $ToDate) }
+if ($TargetId) { $Args += @("--target-id", $TargetId) }
+if ($LimitPerSet -gt 0) { $Args += @("--limit-per-set", $LimitPerSet.ToString()) }
+if ($Output) { $Args += @("--output", $Output) }
+if ($Strict) { $Args += "--strict" }
+foreach ($Name in $EntitySet) {
+  if ($Name) { $Args += @("--entity-set", $Name) }
+}
+foreach ($Item in $Keyword) {
+  if ($Item) { $Args += @("--keyword", $Item) }
+}
+
+& $EnvPython @Args
+exit $LASTEXITCODE

scripts/run_refresh.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.refresh import REFRESH_MODES, RefreshService
+from config.settings import LOGS_DIR
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run canonical refresh into local/remote canonical store",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=sorted(REFRESH_MODES),
+        default="incremental",
+        help="Refresh mode: historical, incremental, targeted",
+    )
+    parser.add_argument("--from-date", dest="date_from", default=None, help="ISO date lower bound")
+    parser.add_argument("--to-date", dest="date_to", default=None, help="ISO date upper bound")
+    parser.add_argument("--target-id", default=None, help="Entity/document id fragment for targeted mode")
+    parser.add_argument(
+        "--limit-per-set",
+        type=int,
+        default=None,
+        help="How many source rows to read per entity set",
+    )
+    parser.add_argument(
+        "--entity-set",
+        action="append",
+        dest="entity_sets",
+        default=None,
+        help="Specific entity set to include (repeat flag for multiple values)",
+    )
+    parser.add_argument(
+        "--keyword",
+        action="append",
+        dest="keywords",
+        default=None,
+        help="Entity-set matcher keyword (repeat flag for multiple values)",
+    )
+    parser.add_argument(
+        "--output",
+        default=str(LOGS_DIR / "refresh_last_run.json"),
+        help="Where to write run summary json",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with code 1 when run status is failed or partial_success",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    service = RefreshService.build()
+    result = service.run_refresh(
+        mode=args.mode,
+        date_from=args.date_from,
+        date_to=args.date_to,
+        target_id=args.target_id,
+        limit_per_set=args.limit_per_set,
+        requested_entity_sets=args.entity_sets,
+        entity_keywords=args.keywords,
+    )
+
+    payload = result.to_dict()
+    payload["store_stats"] = service.store_stats()
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+    if args.strict and result.status != "success":
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_risk.ps1

@@ -0,0 +1,26 @@
+$ErrorActionPreference = "Stop"
+
+param(
+  [string]$SourceFeatureRunId = "",
+  [int]$AnomalyLimit = 0,
+  [string]$Output = "",
+  [switch]$Strict
+)
+
+$EnvName = "ndc_1c_mvp"
+$EnvPython = Join-Path $env:USERPROFILE "miniconda3\envs\$EnvName\python.exe"
+if (-not (Test-Path $EnvPython)) {
+  $EnvPython = Join-Path $env:USERPROFILE "Miniconda3\envs\$EnvName\python.exe"
+}
+if (-not (Test-Path $EnvPython)) {
+  throw "Python for env '$EnvName' not found. Expected: $EnvPython"
+}
+
+$Args = @("scripts/run_risk.py")
+if ($SourceFeatureRunId) { $Args += @("--source-feature-run-id", $SourceFeatureRunId) }
+if ($AnomalyLimit -gt 0) { $Args += @("--anomaly-limit", $AnomalyLimit.ToString()) }
+if ($Output) { $Args += @("--output", $Output) }
+if ($Strict) { $Args += "--strict" }
+
+& $EnvPython @Args
+exit $LASTEXITCODE

scripts/run_risk.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from canonical_layer.risk import RiskService
+from config.settings import LOGS_DIR
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run risk scoring engine from feature/anomaly layer",
+    )
+    parser.add_argument(
+        "--source-feature-run-id",
+        default=None,
+        help="Optional explicit feature run id to score",
+    )
+    parser.add_argument(
+        "--anomaly-limit",
+        type=int,
+        default=None,
+        help="How many anomalies to scan",
+    )
+    parser.add_argument(
+        "--output",
+        default=str(LOGS_DIR / "risk_last_run.json"),
+        help="Where to write run summary json",
+    )
+    parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with code 1 if run status is not success",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    service = RiskService.build()
+    result = service.run_risk_engine(
+        source_feature_run_id=args.source_feature_run_id,
+        anomaly_limit=args.anomaly_limit,
+    )
+
+    payload = result.to_dict()
+    payload["risk_store_stats"] = service.stats()
+    payload["risk_runs"] = service.list_recent_runs(limit=5)
+    payload["risk_patterns"] = service.list_patterns(limit=200, active_only=True)
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps(payload, ensure_ascii=False, indent=2))
+
+    if args.strict and result.status != "success":
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())