Initial import NDC_1C

canonical_layer/__init__.py

@@ -0,0 +1,2 @@
+"""Canonical layer package for normalized accounting entities."""
+

canonical_layer/app.py

@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+from fastapi import FastAPI, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from canonical_layer.features import FeatureService
+from canonical_layer.refresh import REFRESH_MODES, RefreshService
+from canonical_layer.risk import RiskService
+from canonical_layer.service import CanonicalService
+
+
+app = FastAPI(
+    title="1C Canonical Layer MVP",
+    version="0.1.0",
+    description="Read-only API over 1C OData probe and canonical mapping layer.",
+)
+service = CanonicalService.build()
+refresh_service = RefreshService.build()
+feature_service = FeatureService.build()
+risk_service = RiskService.build()
+
+
+class RefreshRunRequest(BaseModel):
+    mode: str = Field(default="incremental")
+    date_from: str | None = None
+    date_to: str | None = None
+    target_id: str | None = None
+    limit_per_set: int | None = Field(default=None, ge=1, le=5000)
+    entity_sets: list[str] | None = None
+    keywords: list[str] | None = None
+
+
+class FeatureRunRequest(BaseModel):
+    baseline_window_hours: int | None = Field(default=None, ge=1, le=720)
+    stale_refresh_threshold_hours: int | None = Field(default=None, ge=1, le=720)
+    top_account_tokens: int = Field(default=20, ge=1, le=200)
+    entity_limit: int | None = Field(default=None, ge=1, le=200000)
+
+
+class RiskRunRequest(BaseModel):
+    source_feature_run_id: str | None = None
+    anomaly_limit: int | None = Field(default=None, ge=1, le=20000)
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {
+        "status": "ok",
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+@app.get("/metadata/entity-sets")
+def metadata_entity_sets() -> dict[str, Any]:
+    return service.list_entity_sets()
+
+
+@app.get("/documents")
+def list_documents(
+    from_date: str | None = Query(default=None, alias="from"),
+    to_date: str | None = Query(default=None, alias="to"),
+    limit: int = Query(default=100, ge=1, le=500),
+) -> dict[str, Any]:
+    items = service.get_documents(date_from=from_date, date_to=to_date, limit=limit)
+    return {
+        "total": len(items),
+        "items": [item.model_dump() for item in items],
+    }
+
+
+@app.get("/documents/{document_id}")
+def get_document(document_id: str) -> dict[str, Any]:
+    document = service.get_document(document_id)
+    if document is None:
+        raise HTTPException(status_code=404, detail="Document not found in current probe window")
+    return document.model_dump()
+
+
+@app.get("/postings")
+def list_postings(
+    account: str | None = Query(default=None),
+    from_date: str | None = Query(default=None, alias="from"),
+    to_date: str | None = Query(default=None, alias="to"),
+    limit: int = Query(default=100, ge=1, le=500),
+) -> dict[str, Any]:
+    items = service.get_postings(account=account, date_from=from_date, date_to=to_date, limit=limit)
+    return {
+        "total": len(items),
+        "items": [item.model_dump() for item in items],
+    }
+
+
+@app.get("/counterparties/{counterparty_id}/documents")
+def counterparty_documents(
+    counterparty_id: str,
+    limit: int = Query(default=100, ge=1, le=500),
+) -> dict[str, Any]:
+    items = service.get_counterparty_documents(counterparty_id=counterparty_id, limit=limit)
+    return {
+        "counterparty_id": counterparty_id,
+        "total": len(items),
+        "items": [item.model_dump() for item in items],
+    }
+
+
+@app.get("/graph/document/{document_id}")
+def document_graph(document_id: str) -> dict[str, Any]:
+    return service.build_document_graph(document_id)
+
+
+@app.get("/store/stats")
+def store_stats() -> dict[str, Any]:
+    return refresh_service.store_stats()
+
+
+@app.get("/refresh/runs")
+def refresh_runs(limit: int = Query(default=20, ge=1, le=200)) -> dict[str, Any]:
+    runs = refresh_service.list_recent_runs(limit=limit)
+    return {
+        "total": len(runs),
+        "items": runs,
+    }
+
+
+@app.post("/refresh/run")
+def run_refresh(request: RefreshRunRequest) -> dict[str, Any]:
+    mode = request.mode.strip().lower()
+    if mode not in REFRESH_MODES:
+        raise HTTPException(status_code=400, detail=f"Unsupported mode '{request.mode}'")
+
+    result = refresh_service.run_refresh(
+        mode=mode,
+        date_from=request.date_from,
+        date_to=request.date_to,
+        target_id=request.target_id,
+        limit_per_set=request.limit_per_set,
+        requested_entity_sets=request.entity_sets,
+        entity_keywords=request.keywords,
+    )
+    payload = result.to_dict()
+    payload["store_stats"] = refresh_service.store_stats()
+    return payload
+
+
+@app.get("/features/stats")
+def feature_stats() -> dict[str, Any]:
+    return feature_service.stats()
+
+
+@app.get("/features/runs")
+def feature_runs(limit: int = Query(default=20, ge=1, le=200)) -> dict[str, Any]:
+    runs = feature_service.list_recent_runs(limit=limit)
+    return {
+        "total": len(runs),
+        "items": runs,
+    }
+
+
+@app.get("/features/metrics")
+def feature_metrics(
+    limit: int = Query(default=200, ge=1, le=2000),
+    metric_key: str | None = Query(default=None),
+    scope: str | None = Query(default=None),
+    run_id: str | None = Query(default=None),
+) -> dict[str, Any]:
+    items = feature_service.list_metrics(
+        limit=limit,
+        metric_key=metric_key,
+        scope=scope,
+        run_id=run_id,
+    )
+    return {
+        "total": len(items),
+        "items": items,
+    }
+
+
+@app.get("/features/anomalies")
+def feature_anomalies(
+    limit: int = Query(default=200, ge=1, le=2000),
+    severity: str | None = Query(default=None),
+    active_only: bool = Query(default=True),
+    run_id: str | None = Query(default=None),
+) -> dict[str, Any]:
+    items = feature_service.list_anomalies(
+        limit=limit,
+        severity=severity,
+        active_only=active_only,
+        run_id=run_id,
+    )
+    return {
+        "total": len(items),
+        "items": items,
+    }
+
+
+@app.post("/features/run")
+def run_features(request: FeatureRunRequest) -> dict[str, Any]:
+    result = feature_service.run_feature_engine(
+        baseline_window_hours=request.baseline_window_hours,
+        stale_refresh_threshold_hours=request.stale_refresh_threshold_hours,
+        top_account_tokens=request.top_account_tokens,
+        entity_limit=request.entity_limit,
+    )
+    payload = result.to_dict()
+    payload["feature_store_stats"] = feature_service.stats()
+    payload["store_stats"] = refresh_service.store_stats()
+    return payload
+
+
+@app.get("/risk/stats")
+def risk_stats() -> dict[str, Any]:
+    return risk_service.stats()
+
+
+@app.get("/risk/runs")
+def risk_runs(limit: int = Query(default=20, ge=1, le=200)) -> dict[str, Any]:
+    runs = risk_service.list_recent_runs(limit=limit)
+    return {
+        "total": len(runs),
+        "items": runs,
+    }
+
+
+@app.get("/risk/patterns")
+def risk_patterns(
+    limit: int = Query(default=200, ge=1, le=2000),
+    severity: str | None = Query(default=None),
+    active_only: bool = Query(default=True),
+    run_id: str | None = Query(default=None),
+    pattern_key: str | None = Query(default=None),
+    scope: str | None = Query(default=None),
+) -> dict[str, Any]:
+    items = risk_service.list_patterns(
+        limit=limit,
+        severity=severity,
+        active_only=active_only,
+        run_id=run_id,
+        pattern_key=pattern_key,
+        scope=scope,
+    )
+    return {
+        "total": len(items),
+        "items": items,
+    }
+
+
+@app.post("/risk/run")
+def run_risk(request: RiskRunRequest) -> dict[str, Any]:
+    result = risk_service.run_risk_engine(
+        source_feature_run_id=request.source_feature_run_id,
+        anomaly_limit=request.anomaly_limit,
+    )
+    payload = result.to_dict()
+    payload["risk_store_stats"] = risk_service.stats()
+    payload["feature_store_stats"] = feature_service.stats()
+    payload["store_stats"] = refresh_service.store_stats()
+    return payload

canonical_layer/features.py

@@ -0,0 +1,454 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import json
+import math
+import re
+from typing import Any
+
+from canonical_layer.store import CanonicalStore
+from config.settings import OneCSettings, load_settings
+
+
+ACCOUNT_TOKEN_RE = re.compile(r"\b\d{2}(?:\.\d{2})?\b")
+
+
+@dataclass
+class FeatureEngineResult:
+    run_id: str
+    status: str
+    baseline_window_hours: int
+    stale_refresh_threshold_hours: int
+    entities_total: int
+    metrics_written: int
+    anomalies_written: int
+    error_message: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "status": self.status,
+            "baseline_window_hours": self.baseline_window_hours,
+            "stale_refresh_threshold_hours": self.stale_refresh_threshold_hours,
+            "entities_total": self.entities_total,
+            "metrics_written": self.metrics_written,
+            "anomalies_written": self.anomalies_written,
+            "error_message": self.error_message,
+        }
+
+
+class FeatureService:
+    def __init__(self, *, settings: OneCSettings, store: CanonicalStore) -> None:
+        self.settings = settings
+        self.store = store
+        self.store.ensure_created()
+
+    @classmethod
+    def build(cls) -> "FeatureService":
+        settings = load_settings()
+        store = CanonicalStore(settings.canonical_db_url)
+        return cls(settings=settings, store=store)
+
+    @staticmethod
+    def _stddev(values: list[int]) -> float:
+        if len(values) <= 1:
+            return 0.0
+        mean = sum(values) / len(values)
+        variance = sum((value - mean) ** 2 for value in values) / len(values)
+        return math.sqrt(variance)
+
+    def _latest_previous_successful_run_id(self, *, current_run_id: str) -> str | None:
+        runs = self.store.list_recent_feature_runs(limit=50)
+        for run in runs:
+            if run.get("run_id") == current_run_id:
+                continue
+            if run.get("status") == "success":
+                return str(run.get("run_id"))
+        return None
+
+    def run_feature_engine(
+        self,
+        *,
+        baseline_window_hours: int | None = None,
+        stale_refresh_threshold_hours: int | None = None,
+        top_account_tokens: int = 20,
+        entity_limit: int | None = None,
+    ) -> FeatureEngineResult:
+        baseline_hours = baseline_window_hours or self.settings.feature_default_baseline_window_hours
+        stale_hours = stale_refresh_threshold_hours or self.settings.anomaly_stale_refresh_threshold_hours
+        scan_limit = entity_limit or self.settings.feature_entity_scan_limit
+
+        run_id = self.store.start_feature_run(baseline_window_hours=baseline_hours)
+        now = datetime.now(timezone.utc)
+
+        try:
+            entities = self.store.iter_entities_for_features(limit=scan_limit)
+            link_counts = self.store.link_counts_by_source()
+
+            per_set_count: dict[str, int] = defaultdict(int)
+            per_set_empty_display: dict[str, int] = defaultdict(int)
+            per_set_link_sum: dict[str, int] = defaultdict(int)
+            account_token_count: dict[str, int] = defaultdict(int)
+            entity_link_items: list[dict[str, Any]] = []
+
+            metrics: list[dict[str, Any]] = []
+            anomalies: list[dict[str, Any]] = []
+
+            for entity in entities:
+                source_entity = str(entity.get("source_entity", "unknown"))
+                source_id = str(entity.get("source_id", ""))
+                display_name = str(entity.get("display_name", "")).strip()
+                attributes = entity.get("attributes", {})
+
+                per_set_count[source_entity] += 1
+                if not display_name:
+                    per_set_empty_display[source_entity] += 1
+
+                link_count = int(link_counts.get((source_entity, source_id), 0))
+                per_set_link_sum[source_entity] += link_count
+                entity_link_items.append(
+                    {
+                        "source_entity": source_entity,
+                        "source_id": source_id,
+                        "display_name": display_name,
+                        "link_count": link_count,
+                    }
+                )
+
+                searchable_blob = f"{display_name} {source_id} {json.dumps(attributes, ensure_ascii=False)}"
+                for token in ACCOUNT_TOKEN_RE.findall(searchable_blob):
+                    account_token_count[token] += 1
+
+            entities_total = len(entities)
+            links_total = sum(item["link_count"] for item in entity_link_items)
+            entity_sets_total = len(per_set_count)
+            avg_links_per_entity = (links_total / entities_total) if entities_total else 0.0
+
+            metrics.append(
+                {
+                    "metric_key": "canonical_entities_total",
+                    "scope": "global",
+                    "scope_id": "",
+                    "metric_type": "gauge",
+                    "metric_value": float(entities_total),
+                    "attributes": {},
+                }
+            )
+            metrics.append(
+                {
+                    "metric_key": "canonical_links_total",
+                    "scope": "global",
+                    "scope_id": "",
+                    "metric_type": "gauge",
+                    "metric_value": float(links_total),
+                    "attributes": {},
+                }
+            )
+            metrics.append(
+                {
+                    "metric_key": "canonical_entity_sets_total",
+                    "scope": "global",
+                    "scope_id": "",
+                    "metric_type": "gauge",
+                    "metric_value": float(entity_sets_total),
+                    "attributes": {},
+                }
+            )
+            metrics.append(
+                {
+                    "metric_key": "avg_links_per_entity",
+                    "scope": "global",
+                    "scope_id": "",
+                    "metric_type": "gauge",
+                    "metric_value": float(avg_links_per_entity),
+                    "attributes": {},
+                }
+            )
+
+            if entities_total == 0:
+                anomalies.append(
+                    {
+                        "signal_type": "no_canonical_data",
+                        "severity": "high",
+                        "scope": "global",
+                        "scope_id": "",
+                        "score": 1.0,
+                        "details": {"reason": "canonical_entities table is empty"},
+                    }
+                )
+
+            for source_entity in sorted(per_set_count):
+                count = per_set_count[source_entity]
+                empty_count = per_set_empty_display.get(source_entity, 0)
+                empty_share = (empty_count / count) if count else 0.0
+                avg_links_local = (per_set_link_sum.get(source_entity, 0) / count) if count else 0.0
+
+                metrics.append(
+                    {
+                        "metric_key": "entity_count",
+                        "scope": "source_entity",
+                        "scope_id": source_entity,
+                        "metric_type": "gauge",
+                        "metric_value": float(count),
+                        "attributes": {},
+                    }
+                )
+                metrics.append(
+                    {
+                        "metric_key": "avg_links_per_entity",
+                        "scope": "source_entity",
+                        "scope_id": source_entity,
+                        "metric_type": "gauge",
+                        "metric_value": float(avg_links_local),
+                        "attributes": {},
+                    }
+                )
+                metrics.append(
+                    {
+                        "metric_key": "empty_display_share",
+                        "scope": "source_entity",
+                        "scope_id": source_entity,
+                        "metric_type": "ratio",
+                        "metric_value": float(empty_share),
+                        "attributes": {"empty_count": empty_count, "total": count},
+                    }
+                )
+
+                if count >= 50 and empty_share >= 0.2:
+                    anomalies.append(
+                        {
+                            "signal_type": "empty_display_share_high",
+                            "severity": "medium",
+                            "scope": "source_entity",
+                            "scope_id": source_entity,
+                            "score": float(empty_share),
+                            "details": {"empty_count": empty_count, "total": count},
+                        }
+                    )
+
+            top_tokens = sorted(account_token_count.items(), key=lambda item: (-item[1], item[0]))[:top_account_tokens]
+            for token, token_count in top_tokens:
+                metrics.append(
+                    {
+                        "metric_key": "account_token_frequency",
+                        "scope": "account_token",
+                        "scope_id": token,
+                        "metric_type": "gauge",
+                        "metric_value": float(token_count),
+                        "attributes": {},
+                    }
+                )
+
+            link_values = [item["link_count"] for item in entity_link_items if item["link_count"] > 0]
+            if link_values:
+                mean_links = sum(link_values) / len(link_values)
+                std_links = self._stddev(link_values)
+                high_link_threshold = max(10, int(mean_links + (3.0 * std_links)))
+            else:
+                high_link_threshold = 10
+
+            metrics.append(
+                {
+                    "metric_key": "high_link_threshold",
+                    "scope": "global",
+                    "scope_id": "",
+                    "metric_type": "gauge",
+                    "metric_value": float(high_link_threshold),
+                    "attributes": {},
+                }
+            )
+
+            suspicious = [
+                item for item in entity_link_items
+                if item["link_count"] >= high_link_threshold
+            ]
+            suspicious.sort(key=lambda item: item["link_count"], reverse=True)
+            for item in suspicious[:50]:
+                score = float(item["link_count"]) / float(high_link_threshold) if high_link_threshold else 0.0
+                severity = "high" if score >= 2.0 else "medium"
+                anomalies.append(
+                    {
+                        "signal_type": "high_link_degree",
+                        "severity": severity,
+                        "scope": item["source_entity"],
+                        "scope_id": item["source_id"],
+                        "score": score,
+                        "details": {
+                            "link_count": item["link_count"],
+                            "threshold": high_link_threshold,
+                            "display_name": item["display_name"],
+                        },
+                    }
+                )
+
+            latest_refresh = self.store.latest_refresh_finished_at()
+            if latest_refresh is None:
+                anomalies.append(
+                    {
+                        "signal_type": "missing_refresh_baseline",
+                        "severity": "high",
+                        "scope": "global",
+                        "scope_id": "",
+                        "score": 1.0,
+                        "details": {"reason": "no successful refresh run found"},
+                    }
+                )
+            else:
+                if latest_refresh.tzinfo is None:
+                    latest_refresh = latest_refresh.replace(tzinfo=timezone.utc)
+                age_hours = max(0.0, (now - latest_refresh).total_seconds() / 3600.0)
+                metrics.append(
+                    {
+                        "metric_key": "refresh_age_hours",
+                        "scope": "global",
+                        "scope_id": "",
+                        "metric_type": "gauge",
+                        "metric_value": float(age_hours),
+                        "attributes": {"latest_refresh_finished_at": latest_refresh.isoformat()},
+                    }
+                )
+                if age_hours > stale_hours:
+                    anomalies.append(
+                        {
+                            "signal_type": "stale_refresh",
+                            "severity": "high",
+                            "scope": "global",
+                            "scope_id": "",
+                            "score": float(age_hours / stale_hours) if stale_hours else float(age_hours),
+                            "details": {
+                                "age_hours": age_hours,
+                                "threshold_hours": stale_hours,
+                                "latest_refresh_finished_at": latest_refresh.isoformat(),
+                            },
+                        }
+                    )
+
+            previous_run_id = self._latest_previous_successful_run_id(current_run_id=run_id)
+            if previous_run_id:
+                prev_entity_count_rows = self.store.list_feature_metrics(
+                    limit=5000,
+                    metric_key="entity_count",
+                    scope="source_entity",
+                    run_id=previous_run_id,
+                )
+                prev_counts = {
+                    str(row.get("scope_id", "")): float(row.get("metric_value", 0.0))
+                    for row in prev_entity_count_rows
+                }
+                for source_entity, current_count in per_set_count.items():
+                    previous_count = prev_counts.get(source_entity)
+                    if previous_count is None or previous_count <= 0:
+                        continue
+                    drift_ratio = (float(current_count) - previous_count) / previous_count
+                    metrics.append(
+                        {
+                            "metric_key": "entity_count_drift_ratio",
+                            "scope": "source_entity",
+                            "scope_id": source_entity,
+                            "metric_type": "ratio",
+                            "metric_value": float(drift_ratio),
+                            "attributes": {
+                                "previous_count": previous_count,
+                                "current_count": current_count,
+                                "previous_run_id": previous_run_id,
+                            },
+                        }
+                    )
+                    if abs(drift_ratio) >= 0.3 and abs(float(current_count) - previous_count) >= 10:
+                        anomalies.append(
+                            {
+                                "signal_type": "entity_count_drift",
+                                "severity": "high" if abs(drift_ratio) >= 1.0 else "medium",
+                                "scope": "source_entity",
+                                "scope_id": source_entity,
+                                "score": float(abs(drift_ratio)),
+                                "details": {
+                                    "previous_count": previous_count,
+                                    "current_count": current_count,
+                                    "drift_ratio": drift_ratio,
+                                    "previous_run_id": previous_run_id,
+                                },
+                            }
+                        )
+
+            metrics_written, anomalies_written = self.store.replace_feature_results(
+                run_id=run_id,
+                metrics=metrics,
+                anomalies=anomalies,
+            )
+
+            details = {
+                "entity_sets_total": entity_sets_total,
+                "top_account_tokens": [{"token": token, "count": count} for token, count in top_tokens],
+            }
+            self.store.finish_feature_run(
+                run_id=run_id,
+                status="success",
+                entities_total=entities_total,
+                metrics_written=metrics_written,
+                anomalies_written=anomalies_written,
+                details=details,
+            )
+
+            return FeatureEngineResult(
+                run_id=run_id,
+                status="success",
+                baseline_window_hours=baseline_hours,
+                stale_refresh_threshold_hours=stale_hours,
+                entities_total=entities_total,
+                metrics_written=metrics_written,
+                anomalies_written=anomalies_written,
+            )
+        except Exception as exc:
+            self.store.finish_feature_run(
+                run_id=run_id,
+                status="failed",
+                entities_total=0,
+                metrics_written=0,
+                anomalies_written=0,
+                details={},
+                error_message=str(exc),
+            )
+            return FeatureEngineResult(
+                run_id=run_id,
+                status="failed",
+                baseline_window_hours=baseline_hours,
+                stale_refresh_threshold_hours=stale_hours,
+                entities_total=0,
+                metrics_written=0,
+                anomalies_written=0,
+                error_message=str(exc),
+            )
+
+    def list_recent_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        return self.store.list_recent_feature_runs(limit=limit)
+
+    def list_metrics(
+        self,
+        *,
+        limit: int = 200,
+        metric_key: str | None = None,
+        scope: str | None = None,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        return self.store.list_feature_metrics(limit=limit, metric_key=metric_key, scope=scope, run_id=run_id)
+
+    def list_anomalies(
+        self,
+        *,
+        limit: int = 200,
+        severity: str | None = None,
+        active_only: bool = True,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        return self.store.list_anomaly_signals(
+            limit=limit,
+            severity=severity,
+            active_only=active_only,
+            run_id=run_id,
+        )
+
+    def stats(self) -> dict[str, Any]:
+        return self.store.feature_store_stats()

canonical_layer/mappers.py

@@ -0,0 +1,472 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from typing import Any
+
+from canonical_layer.models import (
+    Account,
+    BankAccount,
+    CanonicalEntity,
+    CashflowArticle,
+    Contract,
+    Counterparty,
+    Currency,
+    Department,
+    Document,
+    EntityLink,
+    Individual,
+    InvoiceDocument,
+    Item,
+    Organization,
+    Period,
+    Posting,
+    RegisterMovement,
+    RegisterRecord,
+    ResponsiblePerson,
+    Subconto,
+    Warehouse,
+)
+
+
+GUID_RE = re.compile(
+    r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
+)
+ZERO_GUID = "00000000-0000-0000-0000-000000000000"
+MISSING_SOURCE_IDS = {"", "unknown", "none", "null", "n/a", "nan"}
+SOURCE_ID_FIELDS = ("Ref_Key", "Ref", "ID", "Id", "id", "Key")
+DISPLAY_FIELDS = (
+    "Description",
+    "Presentation",
+    "Number",
+    "Code",
+    "Наименование",
+    "Представление",
+)
+
+
+def _normalize_text(value: Any) -> str:
+    return str(value or "").strip()
+
+
+def _normalize_key(value: str) -> str:
+    lowered = value.strip().lower()
+    return re.sub(r"[^a-zа-я0-9_]+", "", lowered)
+
+
+def _is_guid(value: Any) -> bool:
+    if not isinstance(value, str):
+        return False
+    return bool(GUID_RE.match(value.strip()))
+
+
+def _is_zero_guid(value: str) -> bool:
+    return value.strip().lower() == ZERO_GUID
+
+
+def _pick_first(record: dict[str, Any], field_names: tuple[str, ...], default: str) -> str:
+    for field in field_names:
+        value = record.get(field)
+        normalized = _normalize_text(value)
+        if normalized:
+            return normalized
+    return default
+
+
+def _reference_type_field(record: dict[str, Any], field: str) -> tuple[str | None, str | None]:
+    candidates = (
+        f"{field}_Type",
+        f"{field}Type",
+    )
+    for candidate in candidates:
+        if candidate in record:
+            text = _normalize_text(record.get(candidate))
+            if text:
+                return candidate, text
+    return None, None
+
+
+def _build_composite_source_id(entity_set: str, record: dict[str, Any]) -> str:
+    composite_payload = {
+        "entity_set": entity_set,
+        "Recorder": _normalize_text(record.get("Recorder")),
+        "Recorder_Type": _normalize_text(record.get("Recorder_Type")),
+        "Ref": _normalize_text(record.get("Ref")),
+        "Ref_Type": _normalize_text(record.get("Ref_Type")),
+        "LineNumber": _normalize_text(record.get("LineNumber")),
+        "Period": _normalize_text(record.get("Period")),
+        "Date": _normalize_text(record.get("Date")),
+    }
+    digest = hashlib.sha1(
+        json.dumps(composite_payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
+    ).hexdigest()
+    return f"cmp:{digest}"
+
+
+def _guess_entity_from_type_hint(type_hint: str) -> str | None:
+    text = _normalize_key(type_hint)
+    if not text:
+        return None
+
+    if "document_" in text or "документ" in text:
+        if "счетфактур" in text or "invoice" in text:
+            return "InvoiceDocument"
+        return "Document"
+    if "catalog_" in text or "справочник" in text:
+        if "контрагент" in text or "counterparty" in text:
+            return "Counterparty"
+        if "договор" in text or "contract" in text:
+            return "Contract"
+        if "валют" in text or "currency" in text:
+            return "Currency"
+        if "склад" in text or "warehouse" in text:
+            return "Warehouse"
+        if "физическоелицо" in text or "физлиц" in text or "individual" in text:
+            return "Individual"
+        if "статьядвиженияденежныхсредств" in text or "cashflow" in text:
+            return "CashflowArticle"
+        if "подраздел" in text or "department" in text:
+            return "Department"
+        if "номенклатур" in text or "item" in text or "product" in text:
+            return "Item"
+        if "пользоват" in text or "сотрудник" in text or "employee" in text or "user" in text:
+            return "ResponsiblePerson"
+        if "банковскиесчета" in text or "bankaccount" in text:
+            return "BankAccount"
+        if "организац" in text or "organization" in text:
+            return "Organization"
+        if "счета" in text or "account" in text:
+            return "Account"
+    if "accumulationregister_" in text or "informationregister_" in text or "accountingregister_" in text:
+        return "RegisterRecord"
+    return None
+
+
+def _is_document_journal(entity_set: str) -> bool:
+    lowered = entity_set.lower()
+    return "documentjournal_" in lowered or "журнал" in lowered
+
+
+def _is_register_entity_set(entity_set: str) -> bool:
+    lowered = entity_set.lower()
+    return (
+        "register" in lowered
+        or "регистр" in lowered
+        or "accumulation" in lowered
+        or "informationregister_" in lowered
+        or "accountingregister_" in lowered
+    )
+
+
+def _is_document_entity_set(entity_set: str) -> bool:
+    lowered = entity_set.lower()
+    return "document_" in lowered or "document" in lowered or "документ" in lowered
+
+
+def _field_role(field: str) -> str | None:
+    key = _normalize_key(field)
+    if key in {"recorder", "регистратор"}:
+        return "recorder"
+    if key == "ref":
+        return "ref"
+    if "счетфактур" in key or "invoice" in key:
+        return "invoice"
+    if "поставщик" in key or "supplier" in key:
+        return "supplier"
+    if "покупатель" in key or "buyer" in key or "customer" in key:
+        return "buyer"
+    if "контрагент" in key or "counterparty" in key:
+        return "counterparty"
+    if "договор" in key or "contract" in key:
+        return "contract"
+    if "организац" in key or "organization" in key:
+        return "organization"
+    if "ответствен" in key or "responsible" in key:
+        return "responsible"
+    if "валют" in key or "currency" in key:
+        return "currency"
+    if "склад" in key or "warehouse" in key:
+        return "warehouse"
+    if "статьядвиженияденежныхсредств" in key or "cashflow" in key:
+        return "cashflow_article"
+    if "физлиц" in key or "individual" in key or "person" in key:
+        return "individual"
+    if "подраздел" in key or "department" in key:
+        return "department"
+    if "банковскисчет" in key or "банковскийсчет" in key or "bankaccount" in key:
+        return "bank_account"
+    if "счеторганизац" in key or "organizationaccount" in key:
+        return "bank_account"
+    if "номенклатур" in key or "товар" in key or "item" in key or "product" in key:
+        return "item"
+    if "счет" in key or "account" in key:
+        return "account"
+    return None
+
+
+def _role_target(role: str) -> str:
+    mapping = {
+        "recorder": "Document",
+        "ref": "Document",
+        "invoice": "InvoiceDocument",
+        "supplier": "Counterparty",
+        "buyer": "Counterparty",
+        "counterparty": "Counterparty",
+        "contract": "Contract",
+        "organization": "Organization",
+        "responsible": "ResponsiblePerson",
+        "currency": "Currency",
+        "warehouse": "Warehouse",
+        "cashflow_article": "CashflowArticle",
+        "individual": "Individual",
+        "department": "Department",
+        "bank_account": "BankAccount",
+        "item": "Item",
+        "account": "Account",
+    }
+    return mapping.get(role, "Unknown")
+
+
+def _role_relation(entity_set: str, role: str) -> str:
+    if _is_document_journal(entity_set):
+        if role == "ref":
+            return "journal_refers_to_document"
+        if role == "currency":
+            return "journal_has_currency"
+        return f"journal_{role}"
+
+    if _is_register_entity_set(entity_set):
+        mapping = {
+            "recorder": "register_recorded_by_document",
+            "invoice": "register_relates_to_invoice",
+            "supplier": "register_relates_to_supplier",
+            "buyer": "register_relates_to_buyer",
+            "counterparty": "register_relates_to_counterparty",
+            "contract": "register_relates_to_contract",
+            "organization": "register_relates_to_organization",
+            "currency": "register_relates_to_currency",
+            "warehouse": "register_relates_to_warehouse",
+            "bank_account": "register_relates_to_bank_account",
+            "item": "register_relates_to_item",
+            "account": "register_relates_to_account",
+            "department": "register_relates_to_department",
+            "individual": "register_relates_to_individual",
+            "cashflow_article": "register_relates_to_cashflow_article",
+            "responsible": "register_has_responsible",
+        }
+        return mapping.get(role, "register_reference")
+
+    if _is_document_entity_set(entity_set):
+        mapping = {
+            "counterparty": "document_has_counterparty",
+            "contract": "document_has_contract",
+            "organization": "document_belongs_to_organization",
+            "responsible": "document_has_responsible",
+            "currency": "document_has_currency",
+            "warehouse": "document_has_warehouse",
+            "cashflow_article": "document_has_cashflow_article",
+            "bank_account": "document_has_bank_account",
+            "department": "document_has_department",
+            "individual": "document_relates_to_individual",
+            "invoice": "document_relates_to_invoice",
+            "item": "document_line_has_item",
+            "account": "document_line_has_account",
+            "supplier": "document_has_supplier",
+            "buyer": "document_has_buyer",
+        }
+        return mapping.get(role, "document_reference")
+
+    return "reference"
+
+
+def _guess_target_entity(field: str) -> str:
+    role = _field_role(field)
+    if role is None:
+        return "Unknown"
+    return _role_target(role)
+
+
+def _is_reference_candidate(record: dict[str, Any], field: str, value: Any) -> bool:
+    if field in SOURCE_ID_FIELDS and field != "Ref":
+        return False
+    if field.endswith("@navigationLinkUrl"):
+        return False
+    if field.endswith("_Type"):
+        return False
+    if isinstance(value, (dict, list)):
+        return False
+
+    normalized = _normalize_text(value)
+    if not normalized:
+        return False
+
+    if field.endswith("_Key"):
+        return True
+    if field.lower().endswith("ref"):
+        return True
+    if _is_guid(normalized):
+        return True
+    if _field_role(field) is not None:
+        return True
+    type_field, _ = _reference_type_field(record, field)
+    if type_field:
+        return True
+    return False
+
+
+def _resolve_relation_and_target(
+    *,
+    entity_set: str,
+    field: str,
+    value: str,
+    record: dict[str, Any],
+) -> tuple[str, str]:
+    role = _field_role(field)
+    type_field, type_hint = _reference_type_field(record, field)
+    target_from_type = _guess_entity_from_type_hint(type_hint or "")
+
+    if type_field and role in {"recorder", "ref"} and target_from_type == "InvoiceDocument":
+        # Recorder/Ref should still point at document-level nodes.
+        target_from_type = "Document"
+
+    if role is None:
+        relation = "reference"
+    else:
+        relation = _role_relation(entity_set, role)
+
+    if target_from_type:
+        target_entity = target_from_type
+    elif role is not None:
+        target_entity = _role_target(role)
+    else:
+        target_entity = _guess_target_entity(field)
+
+    if target_entity == "Unknown":
+        relation = "reference"
+
+    if _is_zero_guid(value):
+        return "null_reference", target_entity
+
+    return relation, target_entity
+
+
+def _extract_links(entity_set: str, record: dict[str, Any]) -> list[EntityLink]:
+    links: list[EntityLink] = []
+    for field, raw_value in record.items():
+        if not _is_reference_candidate(record, field, raw_value):
+            continue
+        text_value = _normalize_text(raw_value)
+        if not text_value:
+            continue
+        if _is_zero_guid(text_value):
+            # Keep empty/null references out of canonical graph relations.
+            continue
+
+        relation, target_entity = _resolve_relation_and_target(
+            entity_set=entity_set,
+            field=field,
+            value=text_value,
+            record=record,
+        )
+        links.append(
+            EntityLink(
+                relation=relation,
+                target_entity=target_entity,
+                target_id=text_value,
+                source_field=field,
+            )
+        )
+    return links
+
+
+def _entity_cls_for_set(entity_set: str) -> type[CanonicalEntity]:
+    lowered = entity_set.lower()
+    if "счетфактур" in lowered or "invoice" in lowered:
+        return InvoiceDocument
+    if "документ" in lowered or "document" in lowered:
+        return Document
+    if "контраг" in lowered or "counterparty" in lowered:
+        return Counterparty
+    if "договор" in lowered or "contract" in lowered:
+        return Contract
+    if "банковск" in lowered and "счет" in lowered:
+        return BankAccount
+    if "валют" in lowered or "currency" in lowered:
+        return Currency
+    if "склад" in lowered or "warehouse" in lowered:
+        return Warehouse
+    if "подраздел" in lowered or "department" in lowered:
+        return Department
+    if "физлиц" in lowered or "individual" in lowered:
+        return Individual
+    if "номенклатур" in lowered or "item" in lowered or "product" in lowered:
+        return Item
+    if "ответствен" in lowered or "пользоват" in lowered or "employee" in lowered:
+        return ResponsiblePerson
+    if "статьядвиженияденежныхсредств" in lowered or "cashflow" in lowered:
+        return CashflowArticle
+    if "счет" in lowered or "account" in lowered:
+        return Account
+    if "субконто" in lowered or "subconto" in lowered:
+        return Subconto
+    if "движ" in lowered or "movement" in lowered:
+        return RegisterMovement
+    if "провод" in lowered or "posting" in lowered:
+        return Posting
+    if "регистр" in lowered or "register" in lowered:
+        return RegisterRecord
+    if "период" in lowered or "period" in lowered:
+        return Period
+    if "организ" in lowered or "organization" in lowered:
+        return Organization
+    return CanonicalEntity
+
+
+def _normalize_source_id(value: Any) -> str:
+    text = _normalize_text(value)
+    if text.lower() in MISSING_SOURCE_IDS:
+        return ""
+    return text
+
+
+def map_record(entity_set: str, record: dict[str, Any]) -> CanonicalEntity:
+    source_id = _normalize_source_id(_pick_first(record, SOURCE_ID_FIELDS, default=""))
+    if not source_id:
+        source_id = _build_composite_source_id(entity_set, record)
+
+    display_name = _pick_first(record, DISPLAY_FIELDS, default=source_id)
+    canonical_cls = _entity_cls_for_set(entity_set)
+    return canonical_cls(
+        source_entity=entity_set,
+        source_id=source_id,
+        display_name=display_name,
+        attributes=record,
+        links=_extract_links(entity_set, record),
+    )
+
+
+def map_records(entity_set: str, records: list[dict[str, Any]]) -> list[CanonicalEntity]:
+    return [map_record(entity_set, record) for record in records]
+
+
+def canonical_relation_rule_catalog() -> list[dict[str, str]]:
+    return [
+        {"context": "register", "role": "recorder", "relation": "register_recorded_by_document"},
+        {"context": "journal", "role": "ref", "relation": "journal_refers_to_document"},
+        {"context": "document", "role": "counterparty", "relation": "document_has_counterparty"},
+        {"context": "document", "role": "contract", "relation": "document_has_contract"},
+        {"context": "document", "role": "organization", "relation": "document_belongs_to_organization"},
+        {"context": "document", "role": "responsible", "relation": "document_has_responsible"},
+        {"context": "document", "role": "currency", "relation": "document_has_currency"},
+        {"context": "document", "role": "warehouse", "relation": "document_has_warehouse"},
+        {"context": "document", "role": "cashflow_article", "relation": "document_has_cashflow_article"},
+        {"context": "document", "role": "bank_account", "relation": "document_has_bank_account"},
+        {"context": "register", "role": "supplier", "relation": "register_relates_to_supplier"},
+        {"context": "register", "role": "buyer", "relation": "register_relates_to_buyer"},
+        {"context": "register", "role": "invoice", "relation": "register_relates_to_invoice"},
+        {"context": "register", "role": "contract", "relation": "register_relates_to_contract"},
+        {"context": "register", "role": "organization", "relation": "register_relates_to_organization"},
+        {"context": "register", "role": "account", "relation": "register_relates_to_account"},
+        {"context": "register", "role": "item", "relation": "register_relates_to_item"},
+    ]

canonical_layer/models.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class EntityLink(BaseModel):
+    relation: str
+    target_entity: str
+    target_id: str
+    source_field: str | None = None
+
+
+class CanonicalEntity(BaseModel):
+    source_entity: str
+    source_id: str
+    display_name: str
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    links: list[EntityLink] = Field(default_factory=list)
+
+
+class Organization(CanonicalEntity):
+    pass
+
+
+class Counterparty(CanonicalEntity):
+    pass
+
+
+class Contract(CanonicalEntity):
+    pass
+
+
+class Account(CanonicalEntity):
+    pass
+
+
+class Subconto(CanonicalEntity):
+    pass
+
+
+class ResponsiblePerson(CanonicalEntity):
+    pass
+
+
+class Currency(CanonicalEntity):
+    pass
+
+
+class Warehouse(CanonicalEntity):
+    pass
+
+
+class CashflowArticle(CanonicalEntity):
+    pass
+
+
+class Department(CanonicalEntity):
+    pass
+
+
+class Individual(CanonicalEntity):
+    pass
+
+
+class Item(CanonicalEntity):
+    pass
+
+
+class BankAccount(CanonicalEntity):
+    pass
+
+
+class Document(CanonicalEntity):
+    pass
+
+
+class InvoiceDocument(Document):
+    pass
+
+
+class Posting(CanonicalEntity):
+    pass
+
+
+class RegisterMovement(CanonicalEntity):
+    pass
+
+
+class RegisterRecord(CanonicalEntity):
+    pass
+
+
+class Period(CanonicalEntity):
+    pass

canonical_layer/period_snapshot.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+import re
+from typing import Any
+
+
+ODATA_DATE_RE = re.compile(r"/Date\((?P<millis>-?\d+)")
+DATE_CANDIDATES = ("Date", "Дата", "Period", "Период", "PostedAt", "posted_at")
+
+
+def normalize_dt(value: datetime) -> datetime:
+    if value.tzinfo is None:
+        return value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc)
+
+
+def parse_dt(raw: str | None) -> datetime | None:
+    if raw is None:
+        return None
+    text = raw.strip()
+    if not text:
+        return None
+
+    match = ODATA_DATE_RE.search(text)
+    if match:
+        millis = int(match.group("millis"))
+        return datetime.fromtimestamp(millis / 1000, tz=timezone.utc)
+
+    for candidate in (text.replace("Z", "+00:00"), text):
+        try:
+            return normalize_dt(datetime.fromisoformat(candidate))
+        except ValueError:
+            continue
+    return None
+
+
+def parse_record_datetime(record: dict[str, Any]) -> datetime | None:
+    for field in DATE_CANDIDATES:
+        value = record.get(field)
+        if value is None:
+            continue
+        parsed = parse_dt(str(value))
+        if parsed is not None:
+            return parsed
+    return None
+
+
+def first_day_of_month(value: datetime) -> datetime:
+    normalized = normalize_dt(value)
+    return normalized.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+
+def month_bounds(value: datetime) -> tuple[datetime, datetime]:
+    start = first_day_of_month(value)
+    if start.month == 12:
+        end = start.replace(year=start.year + 1, month=1)
+    else:
+        end = start.replace(month=start.month + 1)
+    return start, end
+
+
+def iso_week_bounds(value: datetime) -> tuple[datetime, datetime]:
+    normalized = normalize_dt(value).replace(hour=0, minute=0, second=0, microsecond=0)
+    start = normalized - timedelta(days=normalized.weekday())
+    end = start + timedelta(days=7)
+    return start, end
+
+
+def window_key(value: datetime, *, granularity: str) -> str:
+    dt = normalize_dt(value)
+    if granularity == "month":
+        return dt.strftime("%Y-%m")
+    if granularity == "week":
+        year, week, _ = dt.isocalendar()
+        return f"{year}-W{week:02d}"
+    raise ValueError(f"Unsupported granularity: {granularity}")
+
+
+def window_bounds_from_key(key: str, *, granularity: str) -> tuple[datetime, datetime]:
+    if granularity == "month":
+        start = normalize_dt(datetime.strptime(key, "%Y-%m"))
+        if start.month == 12:
+            end = start.replace(year=start.year + 1, month=1)
+        else:
+            end = start.replace(month=start.month + 1)
+        return start, end
+    if granularity == "week":
+        year_raw, week_raw = key.split("-W")
+        year = int(year_raw)
+        week = int(week_raw)
+        start = normalize_dt(datetime.fromisocalendar(year, week, 1))
+        end = start + timedelta(days=7)
+        return start, end
+    raise ValueError(f"Unsupported granularity: {granularity}")

canonical_layer/refresh.py

@@ -0,0 +1,296 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+import json
+import re
+from typing import Any
+
+from canonical_layer.mappers import map_records
+from canonical_layer.store import CanonicalStore
+from config.client import ODataClient, extract_entity_sets
+from config.settings import LOGS_DIR, OneCSettings, load_settings
+
+
+ODATA_DATE_RE = re.compile(r"/Date\((?P<millis>-?\d+)")
+REFRESH_MODES = {"historical", "incremental", "targeted"}
+
+
+def _parse_dt(raw: str | None) -> datetime | None:
+    if raw is None:
+        return None
+    text = raw.strip()
+    if not text:
+        return None
+
+    match = ODATA_DATE_RE.search(text)
+    if match:
+        millis = int(match.group("millis"))
+        return datetime.fromtimestamp(millis / 1000)
+
+    for candidate in (text.replace("Z", "+00:00"), text):
+        try:
+            return datetime.fromisoformat(candidate)
+        except ValueError:
+            continue
+    return None
+
+
+def _in_date_range(record: dict[str, Any], date_from: datetime | None, date_to: datetime | None) -> bool:
+    if date_from is None and date_to is None:
+        return True
+
+    date_candidates = ("Date", "Дата", "Period", "Период", "PostedAt", "posted_at")
+    record_dt: datetime | None = None
+    for field in date_candidates:
+        value = record.get(field)
+        if value is None:
+            continue
+        record_dt = _parse_dt(str(value))
+        if record_dt is not None:
+            break
+
+    if record_dt is None:
+        return True
+    if date_from and record_dt < date_from:
+        return False
+    if date_to and record_dt > date_to:
+        return False
+    return True
+
+
+def _unique_preserve_order(items: list[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for item in items:
+        cleaned = item.strip()
+        if not cleaned or cleaned in seen:
+            continue
+        seen.add(cleaned)
+        result.append(cleaned)
+    return result
+
+
+@dataclass
+class RefreshResult:
+    run_id: str
+    mode: str
+    status: str
+    requested_entity_sets: list[str]
+    successful_entity_sets: list[str]
+    failed_entity_sets: list[dict[str, str]]
+    date_from: str | None
+    date_to: str | None
+    target_id: str | None
+    limit_per_set: int
+    records_read: int
+    entities_written: int
+    links_written: int
+    checkpoints_updated: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "mode": self.mode,
+            "status": self.status,
+            "requested_entity_sets": self.requested_entity_sets,
+            "successful_entity_sets": self.successful_entity_sets,
+            "failed_entity_sets": self.failed_entity_sets,
+            "date_from": self.date_from,
+            "date_to": self.date_to,
+            "target_id": self.target_id,
+            "limit_per_set": self.limit_per_set,
+            "records_read": self.records_read,
+            "entities_written": self.entities_written,
+            "links_written": self.links_written,
+            "checkpoints_updated": self.checkpoints_updated,
+        }
+
+
+class RefreshService:
+    def __init__(self, *, settings: OneCSettings, client: ODataClient, store: CanonicalStore) -> None:
+        self.settings = settings
+        self.client = client
+        self.store = store
+        self.store.ensure_created()
+
+    @classmethod
+    def build(cls) -> "RefreshService":
+        settings = load_settings()
+        client = ODataClient(settings)
+        store = CanonicalStore(settings.canonical_db_url)
+        return cls(settings=settings, client=client, store=store)
+
+    def _load_entity_sets(self) -> list[dict[str, str]]:
+        entity_sets_file = LOGS_DIR / "entity_sets.json"
+        if entity_sets_file.exists():
+            payload = json.loads(entity_sets_file.read_text(encoding="utf-8"))
+            entity_sets = payload.get("entity_sets", [])
+            if isinstance(entity_sets, list):
+                return [item for item in entity_sets if isinstance(item, dict)]
+
+        metadata_file = LOGS_DIR / "metadata.xml"
+        if metadata_file.exists():
+            metadata_xml = metadata_file.read_text(encoding="utf-8")
+            return extract_entity_sets(metadata_xml)
+
+        metadata_xml = self.client.fetch_metadata()
+        metadata_file.parent.mkdir(parents=True, exist_ok=True)
+        metadata_file.write_text(metadata_xml, encoding="utf-8")
+        return extract_entity_sets(metadata_xml)
+
+    def _resolve_entity_sets(
+        self,
+        *,
+        requested_entity_sets: list[str] | None,
+        entity_keywords: list[str] | None,
+    ) -> list[str]:
+        if requested_entity_sets:
+            return _unique_preserve_order(requested_entity_sets)
+
+        entity_sets = self._load_entity_sets()
+        names = [str(item.get("name", "")).strip() for item in entity_sets if str(item.get("name", "")).strip()]
+        if not names:
+            return []
+
+        keywords_source = entity_keywords or list(self.settings.refresh_default_entity_keywords)
+        keywords = [keyword.strip().lower() for keyword in keywords_source if keyword.strip()]
+        if not keywords:
+            return names
+
+        matched = [name for name in names if any(keyword in name.lower() for keyword in keywords)]
+        if matched:
+            return _unique_preserve_order(matched)
+        return names
+
+    def run_refresh(
+        self,
+        *,
+        mode: str,
+        date_from: str | None = None,
+        date_to: str | None = None,
+        target_id: str | None = None,
+        limit_per_set: int | None = None,
+        requested_entity_sets: list[str] | None = None,
+        entity_keywords: list[str] | None = None,
+    ) -> RefreshResult:
+        normalized_mode = mode.strip().lower()
+        if normalized_mode not in REFRESH_MODES:
+            raise ValueError(f"Unsupported refresh mode: {mode}")
+
+        resolved_limit = limit_per_set or self.settings.refresh_default_limit_per_set
+        resolved_sets = self._resolve_entity_sets(
+            requested_entity_sets=requested_entity_sets,
+            entity_keywords=entity_keywords,
+        )
+        if not resolved_sets:
+            raise RuntimeError("No entity sets resolved for refresh")
+
+        run_id = self.store.start_refresh_run(
+            mode=normalized_mode,
+            requested_entity_sets=resolved_sets,
+            date_from=date_from,
+            date_to=date_to,
+            limit_per_set=resolved_limit,
+        )
+
+        parsed_date_from = _parse_dt(date_from)
+        parsed_date_to = _parse_dt(date_to)
+        safe_target = target_id.strip() if target_id else None
+
+        successful_sets: list[str] = []
+        failed_sets: list[dict[str, str]] = []
+        records_read = 0
+        entities_written = 0
+        links_written = 0
+
+        per_set_stats: list[dict[str, Any]] = []
+        for entity_set in resolved_sets:
+            try:
+                records = self.client.read_entity_set_records(entity_set, top=resolved_limit)
+            except Exception as exc:
+                failed_sets.append({"entity_set": entity_set, "error": str(exc)})
+                continue
+
+            records_read += len(records)
+            filtered_records = records
+            if parsed_date_from or parsed_date_to:
+                filtered_records = [
+                    row for row in filtered_records if _in_date_range(row, parsed_date_from, parsed_date_to)
+                ]
+
+            if normalized_mode == "targeted" and safe_target:
+                lowered_target = safe_target.lower()
+                filtered_records = [
+                    row
+                    for row in filtered_records
+                    if lowered_target in json.dumps(row, ensure_ascii=False).lower()
+                ]
+
+            mapped = map_records(entity_set, filtered_records)
+            entity_delta, link_delta = self.store.upsert_entities(run_id=run_id, entities=mapped)
+            entities_written += entity_delta
+            links_written += link_delta
+            successful_sets.append(entity_set)
+            per_set_stats.append(
+                {
+                    "entity_set": entity_set,
+                    "records_read": len(records),
+                    "records_after_filters": len(filtered_records),
+                    "entities_written": entity_delta,
+                    "links_written": link_delta,
+                }
+            )
+
+        checkpoints_updated = self.store.update_checkpoints(
+            run_id=run_id,
+            entity_sets=successful_sets,
+            date_from=date_from,
+            date_to=date_to,
+        )
+
+        status = "success"
+        error_message: str | None = None
+        if successful_sets and failed_sets:
+            status = "partial_success"
+        elif failed_sets and not successful_sets:
+            status = "failed"
+            error_message = "; ".join(f"{item['entity_set']}: {item['error']}" for item in failed_sets[:3])
+
+        details = {
+            "per_set": per_set_stats,
+            "failed_sets": failed_sets,
+        }
+        self.store.finish_refresh_run(
+            run_id=run_id,
+            status=status,
+            records_read=records_read,
+            entities_written=entities_written,
+            links_written=links_written,
+            checkpoints_updated=checkpoints_updated,
+            details=details,
+            error_message=error_message,
+        )
+
+        return RefreshResult(
+            run_id=run_id,
+            mode=normalized_mode,
+            status=status,
+            requested_entity_sets=resolved_sets,
+            successful_entity_sets=successful_sets,
+            failed_entity_sets=failed_sets,
+            date_from=date_from,
+            date_to=date_to,
+            target_id=safe_target,
+            limit_per_set=resolved_limit,
+            records_read=records_read,
+            entities_written=entities_written,
+            links_written=links_written,
+            checkpoints_updated=checkpoints_updated,
+        )
+
+    def list_recent_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        return self.store.list_recent_runs(limit=limit)
+
+    def store_stats(self) -> dict[str, Any]:
+        return self.store.store_stats()

canonical_layer/risk.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any
+
+from canonical_layer.store import CanonicalStore
+from config.settings import OneCSettings, load_settings
+
+
+SIGNAL_TO_DOMAIN = {
+    "stale_refresh": "operational_freshness",
+    "missing_refresh_baseline": "operational_freshness",
+    "no_canonical_data": "operational_freshness",
+    "high_link_degree": "suspicious_link_hub",
+    "entity_count_drift": "structural_drift",
+    "empty_display_share_high": "data_quality",
+}
+
+DOMAIN_PATTERN_KEY = {
+    "operational_freshness": "operational_freshness_risk",
+    "suspicious_link_hub": "suspicious_link_hub_risk",
+    "structural_drift": "structural_drift_risk",
+    "data_quality": "data_quality_risk",
+    "miscellaneous": "miscellaneous_risk",
+}
+
+DOMAIN_WEIGHTS = {
+    "operational_freshness": 0.35,
+    "suspicious_link_hub": 0.25,
+    "structural_drift": 0.25,
+    "data_quality": 0.15,
+    "miscellaneous": 0.20,
+}
+
+SEVERITY_BASE = {
+    "low": 0.25,
+    "medium": 0.50,
+    "high": 0.80,
+    "critical": 0.95,
+}
+
+
+@dataclass
+class RiskEngineResult:
+    run_id: str
+    status: str
+    source_feature_run_id: str | None
+    active_anomalies_scanned: int
+    patterns_written: int
+    global_score: float
+    error_message: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "run_id": self.run_id,
+            "status": self.status,
+            "source_feature_run_id": self.source_feature_run_id,
+            "active_anomalies_scanned": self.active_anomalies_scanned,
+            "patterns_written": self.patterns_written,
+            "global_score": self.global_score,
+            "error_message": self.error_message,
+        }
+
+
+class RiskService:
+    def __init__(self, *, settings: OneCSettings, store: CanonicalStore) -> None:
+        self.settings = settings
+        self.store = store
+        self.store.ensure_created()
+
+    @classmethod
+    def build(cls) -> "RiskService":
+        settings = load_settings()
+        store = CanonicalStore(settings.canonical_db_url)
+        return cls(settings=settings, store=store)
+
+    def _severity_from_score(self, score: float) -> str:
+        if score >= self.settings.risk_high_threshold:
+            return "high"
+        if score >= self.settings.risk_medium_threshold:
+            return "medium"
+        return "low"
+
+    @staticmethod
+    def _normalize_score(raw_score: float) -> float:
+        safe = max(0.0, raw_score)
+        if safe <= 1.0:
+            return safe
+        if safe >= 3.0:
+            return 1.0
+        return safe / 3.0
+
+    def _signal_score(self, anomaly: dict[str, Any]) -> float:
+        severity = str(anomaly.get("severity", "medium")).lower()
+        base = SEVERITY_BASE.get(severity, 0.50)
+        raw = self._normalize_score(float(anomaly.get("score", 0.0)))
+        return min(1.0, (0.45 * base) + (0.55 * raw))
+
+    def _build_domain_pattern(self, domain: str, anomalies: list[dict[str, Any]]) -> dict[str, Any]:
+        signal_scores = [self._signal_score(item) for item in anomalies]
+        average_score = (sum(signal_scores) / len(signal_scores)) if signal_scores else 0.0
+        max_score = max(signal_scores) if signal_scores else 0.0
+        density_bonus = min(0.30, max(0, len(anomalies) - 1) * 0.03)
+        domain_score = min(1.0, (0.60 * max_score) + (0.40 * average_score) + density_bonus)
+        confidence = min(1.0, 0.55 + min(0.40, len(anomalies) * 0.05))
+        severity = self._severity_from_score(domain_score)
+
+        by_signal_type: dict[str, int] = defaultdict(int)
+        for item in anomalies:
+            by_signal_type[str(item.get("signal_type", "unknown"))] += 1
+
+        top_examples = sorted(
+            anomalies,
+            key=lambda item: float(item.get("score", 0.0)),
+            reverse=True,
+        )[:5]
+        examples_payload = [
+            {
+                "signal_type": item.get("signal_type"),
+                "severity": item.get("severity"),
+                "scope": item.get("scope"),
+                "scope_id": item.get("scope_id"),
+                "score": item.get("score"),
+                "details": item.get("details", {}),
+            }
+            for item in top_examples
+        ]
+
+        return {
+            "pattern_key": DOMAIN_PATTERN_KEY.get(domain, "miscellaneous_risk"),
+            "severity": severity,
+            "scope": "domain",
+            "scope_id": domain,
+            "score": round(domain_score, 6),
+            "confidence": round(confidence, 6),
+            "details": {
+                "anomalies_count": len(anomalies),
+                "signal_types": dict(sorted(by_signal_type.items(), key=lambda item: item[0])),
+                "top_examples": examples_payload,
+            },
+        }
+
+    def _compute_global_score(self, domain_patterns: list[dict[str, Any]]) -> float:
+        if not domain_patterns:
+            return 0.05
+
+        weighted_sum = 0.0
+        weight_total = 0.0
+        for item in domain_patterns:
+            domain = str(item.get("scope_id", "miscellaneous"))
+            weight = DOMAIN_WEIGHTS.get(domain, DOMAIN_WEIGHTS["miscellaneous"])
+            score = float(item.get("score", 0.0))
+            weighted_sum += weight * score
+            weight_total += weight
+
+        if weight_total <= 0.0:
+            return 0.05
+        return min(1.0, weighted_sum / weight_total)
+
+    def run_risk_engine(
+        self,
+        *,
+        source_feature_run_id: str | None = None,
+        anomaly_limit: int | None = None,
+    ) -> RiskEngineResult:
+        selected_feature_run_id = source_feature_run_id
+        if not selected_feature_run_id:
+            latest = self.store.latest_successful_feature_run()
+            if latest is not None:
+                selected_feature_run_id = str(latest.get("run_id"))
+
+        run_id = self.store.start_risk_run(source_feature_run_id=selected_feature_run_id)
+        safe_limit = anomaly_limit or self.settings.risk_anomaly_scan_limit
+
+        try:
+            if selected_feature_run_id:
+                anomalies = self.store.list_anomaly_signals(
+                    limit=safe_limit,
+                    active_only=False,
+                    run_id=selected_feature_run_id,
+                )
+            else:
+                anomalies = []
+
+            grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+            for anomaly in anomalies:
+                signal_type = str(anomaly.get("signal_type", "unknown"))
+                domain = SIGNAL_TO_DOMAIN.get(signal_type, "miscellaneous")
+                grouped[domain].append(anomaly)
+
+            domain_patterns: list[dict[str, Any]] = []
+            for domain, domain_anomalies in grouped.items():
+                if not domain_anomalies:
+                    continue
+                domain_patterns.append(self._build_domain_pattern(domain, domain_anomalies))
+
+            if not selected_feature_run_id:
+                domain_patterns.append(
+                    {
+                        "pattern_key": DOMAIN_PATTERN_KEY["operational_freshness"],
+                        "severity": "high",
+                        "scope": "domain",
+                        "scope_id": "operational_freshness",
+                        "score": 0.90,
+                        "confidence": 0.95,
+                        "details": {
+                            "anomalies_count": 0,
+                            "signal_types": {},
+                            "top_examples": [],
+                            "reason": "No successful feature run found",
+                        },
+                    }
+                )
+
+            global_score = self._compute_global_score(domain_patterns)
+            global_severity = self._severity_from_score(global_score)
+            global_pattern = {
+                "pattern_key": "global_risk_summary",
+                "severity": global_severity,
+                "scope": "global",
+                "scope_id": "",
+                "score": round(global_score, 6),
+                "confidence": round(min(1.0, 0.60 + 0.05 * len(domain_patterns)), 6),
+                "details": {
+                    "source_feature_run_id": selected_feature_run_id,
+                    "domain_scores": [
+                        {
+                            "domain": item.get("scope_id"),
+                            "score": item.get("score"),
+                            "severity": item.get("severity"),
+                        }
+                        for item in sorted(
+                            domain_patterns,
+                            key=lambda item: float(item.get("score", 0.0)),
+                            reverse=True,
+                        )
+                    ],
+                    "anomalies_scanned": len(anomalies),
+                },
+            }
+
+            all_patterns = [global_pattern] + domain_patterns
+            patterns_written = self.store.replace_risk_patterns(run_id=run_id, patterns=all_patterns)
+            self.store.finish_risk_run(
+                run_id=run_id,
+                status="success",
+                patterns_written=patterns_written,
+                global_score=global_score,
+                details={
+                    "source_feature_run_id": selected_feature_run_id,
+                    "anomalies_scanned": len(anomalies),
+                    "domains_total": len(domain_patterns),
+                },
+            )
+
+            return RiskEngineResult(
+                run_id=run_id,
+                status="success",
+                source_feature_run_id=selected_feature_run_id,
+                active_anomalies_scanned=len(anomalies),
+                patterns_written=patterns_written,
+                global_score=round(global_score, 6),
+            )
+        except Exception as exc:
+            self.store.finish_risk_run(
+                run_id=run_id,
+                status="failed",
+                patterns_written=0,
+                global_score=0.0,
+                details={},
+                error_message=str(exc),
+            )
+            return RiskEngineResult(
+                run_id=run_id,
+                status="failed",
+                source_feature_run_id=selected_feature_run_id,
+                active_anomalies_scanned=0,
+                patterns_written=0,
+                global_score=0.0,
+                error_message=str(exc),
+            )
+
+    def list_recent_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        return self.store.list_recent_risk_runs(limit=limit)
+
+    def list_patterns(
+        self,
+        *,
+        limit: int = 200,
+        severity: str | None = None,
+        active_only: bool = True,
+        run_id: str | None = None,
+        pattern_key: str | None = None,
+        scope: str | None = None,
+    ) -> list[dict[str, Any]]:
+        return self.store.list_risk_patterns(
+            limit=limit,
+            severity=severity,
+            active_only=active_only,
+            run_id=run_id,
+            pattern_key=pattern_key,
+            scope=scope,
+        )
+
+    def stats(self) -> dict[str, Any]:
+        return self.store.risk_store_stats()

canonical_layer/service.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+import json
+from pathlib import Path
+import re
+from typing import Any
+
+from config.client import ODataClient, extract_entity_sets
+from config.settings import LOGS_DIR, load_settings
+from canonical_layer.mappers import map_record, map_records
+from canonical_layer.models import CanonicalEntity
+
+
+ODATA_DATE_RE = re.compile(r"/Date\((?P<millis>-?\d+)")
+
+
+def _parse_dt(raw: str | None) -> datetime | None:
+    if raw is None:
+        return None
+    text = raw.strip()
+    if not text:
+        return None
+
+    match = ODATA_DATE_RE.search(text)
+    if match:
+        millis = int(match.group("millis"))
+        return datetime.fromtimestamp(millis / 1000)
+
+    for candidate in (text.replace("Z", "+00:00"), text):
+        try:
+            return datetime.fromisoformat(candidate)
+        except ValueError:
+            continue
+    return None
+
+
+def _in_date_range(record: dict[str, Any], date_from: datetime | None, date_to: datetime | None) -> bool:
+    if date_from is None and date_to is None:
+        return True
+
+    date_candidates = ("Date", "Дата", "Period", "Период", "PostedAt", "posted_at")
+    record_dt: datetime | None = None
+    for field in date_candidates:
+        value = record.get(field)
+        if value is None:
+            continue
+        record_dt = _parse_dt(str(value))
+        if record_dt is not None:
+            break
+
+    if record_dt is None:
+        return True
+    if date_from and record_dt < date_from:
+        return False
+    if date_to and record_dt > date_to:
+        return False
+    return True
+
+
+@dataclass
+class CanonicalService:
+    client: ODataClient
+
+    @classmethod
+    def build(cls) -> "CanonicalService":
+        settings = load_settings()
+        return cls(client=ODataClient(settings))
+
+    def _load_entity_sets(self) -> list[dict[str, str]]:
+        entity_sets_file = LOGS_DIR / "entity_sets.json"
+        if entity_sets_file.exists():
+            payload = json.loads(entity_sets_file.read_text(encoding="utf-8"))
+            entity_sets = payload.get("entity_sets", [])
+            if isinstance(entity_sets, list):
+                return [item for item in entity_sets if isinstance(item, dict)]
+
+        metadata_file = LOGS_DIR / "metadata.xml"
+        if metadata_file.exists():
+            metadata_xml = metadata_file.read_text(encoding="utf-8")
+            return extract_entity_sets(metadata_xml)
+
+        return []
+
+    def _sets_by_keywords(self, keywords: tuple[str, ...], limit: int = 10) -> list[str]:
+        names: list[str] = []
+        for item in self._load_entity_sets():
+            name = str(item.get("name", ""))
+            lowered = name.lower()
+            if any(keyword in lowered for keyword in keywords):
+                names.append(name)
+        return names[:limit]
+
+    def _safe_read(self, entity_set: str, top: int, extra_params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        try:
+            return self.client.read_entity_set_records(entity_set, top=top, extra_params=extra_params)
+        except Exception:
+            return []
+
+    def list_entity_sets(self) -> dict[str, Any]:
+        entity_sets = self._load_entity_sets()
+        return {
+            "total": len(entity_sets),
+            "entity_sets": entity_sets,
+        }
+
+    def get_documents(self, date_from: str | None, date_to: str | None, limit: int = 100) -> list[CanonicalEntity]:
+        from_dt = _parse_dt(date_from)
+        to_dt = _parse_dt(date_to)
+        document_sets = self._sets_by_keywords(("документ", "document"), limit=5)
+
+        all_records: list[CanonicalEntity] = []
+        per_set_limit = max(3, limit // max(len(document_sets), 1))
+        for entity_set in document_sets:
+            records = self._safe_read(entity_set, top=per_set_limit)
+            filtered = [row for row in records if _in_date_range(row, from_dt, to_dt)]
+            all_records.extend(map_records(entity_set, filtered))
+
+        return all_records[:limit]
+
+    def get_document(self, source_id: str) -> CanonicalEntity | None:
+        document_sets = self._sets_by_keywords(("документ", "document"), limit=8)
+        for entity_set in document_sets:
+            records = self._safe_read(entity_set, top=50)
+            for row in records:
+                for key in ("Ref_Key", "ID", "Id", "id"):
+                    if str(row.get(key, "")).strip() == source_id:
+                        return map_record(entity_set, row)
+        return None
+
+    def get_postings(
+        self,
+        account: str | None,
+        date_from: str | None,
+        date_to: str | None,
+        limit: int = 100,
+    ) -> list[CanonicalEntity]:
+        from_dt = _parse_dt(date_from)
+        to_dt = _parse_dt(date_to)
+        posting_sets = self._sets_by_keywords(("провод", "posting", "хозрасчет", "регистр"), limit=6)
+
+        output: list[CanonicalEntity] = []
+        per_set_limit = max(3, limit // max(len(posting_sets), 1))
+        for entity_set in posting_sets:
+            records = self._safe_read(entity_set, top=per_set_limit)
+            for row in records:
+                if account:
+                    row_string = json.dumps(row, ensure_ascii=False).lower()
+                    if account.lower() not in row_string:
+                        continue
+                if not _in_date_range(row, from_dt, to_dt):
+                    continue
+                output.append(map_record(entity_set, row))
+        return output[:limit]
+
+    def get_counterparty_documents(self, counterparty_id: str, limit: int = 100) -> list[CanonicalEntity]:
+        documents = self.get_documents(date_from=None, date_to=None, limit=limit * 2)
+        result: list[CanonicalEntity] = []
+
+        for doc in documents:
+            if doc.source_id == counterparty_id:
+                continue
+            serialized = json.dumps(doc.attributes, ensure_ascii=False)
+            if counterparty_id in serialized:
+                result.append(doc)
+                continue
+            for link in doc.links:
+                if link.target_id == counterparty_id:
+                    result.append(doc)
+                    break
+
+        return result[:limit]
+
+    def build_document_graph(self, document_id: str) -> dict[str, Any]:
+        root = self.get_document(document_id)
+        if root is None:
+            return {
+                "document_id": document_id,
+                "found": False,
+                "nodes": [],
+                "edges": [],
+            }
+
+        nodes: dict[str, dict[str, Any]] = {
+            root.source_id: {
+                "id": root.source_id,
+                "entity": root.source_entity,
+                "display_name": root.display_name,
+            }
+        }
+        edges: list[dict[str, Any]] = []
+
+        for link in root.links:
+            node_id = link.target_id
+            nodes.setdefault(
+                node_id,
+                {
+                    "id": node_id,
+                    "entity": link.target_entity,
+                    "display_name": node_id,
+                },
+            )
+            edges.append(
+                {
+                    "from": root.source_id,
+                    "to": node_id,
+                    "relation": link.relation,
+                    "field": link.source_field,
+                }
+            )
+
+        postings = self.get_postings(account=None, date_from=None, date_to=None, limit=50)
+        for posting in postings:
+            serialized = json.dumps(posting.attributes, ensure_ascii=False)
+            if document_id not in serialized:
+                continue
+            nodes.setdefault(
+                posting.source_id,
+                {
+                    "id": posting.source_id,
+                    "entity": posting.source_entity,
+                    "display_name": posting.display_name,
+                },
+            )
+            edges.append(
+                {
+                    "from": root.source_id,
+                    "to": posting.source_id,
+                    "relation": "document_to_posting",
+                    "field": "inferred",
+                }
+            )
+
+        return {
+            "document_id": document_id,
+            "found": True,
+            "nodes": list(nodes.values()),
+            "edges": edges,
+        }
+

canonical_layer/store.py

@@ -0,0 +1,744 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import create_engine, delete, func, select
+from sqlalchemy.orm import Session, sessionmaker
+
+from canonical_layer.models import CanonicalEntity
+from canonical_layer.store_models import (
+    AnomalySignalRow,
+    Base,
+    CanonicalEntityRow,
+    CanonicalLinkRow,
+    FeatureMetricRow,
+    FeatureRunRow,
+    RefreshCheckpointRow,
+    RefreshRunRow,
+    RiskPatternRow,
+    RiskRunRow,
+)
+
+
+def _utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _dump_json(payload: Any) -> str:
+    return json.dumps(payload, ensure_ascii=False)
+
+
+def _load_json(payload: str, default: Any) -> Any:
+    if not payload:
+        return default
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError:
+        return default
+
+
+def _dt_to_iso(value: datetime | None) -> str | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc).isoformat()
+
+
+class CanonicalStore:
+    def __init__(self, db_url: str) -> None:
+        self.db_url = db_url
+        self.engine = create_engine(db_url, future=True)
+        self.session_factory = sessionmaker(bind=self.engine, autoflush=False, expire_on_commit=False, future=True)
+
+    def _ensure_sqlite_path(self) -> None:
+        if not self.db_url.startswith("sqlite:///"):
+            return
+        db_path_raw = self.db_url.replace("sqlite:///", "", 1)
+        db_path = Path(db_path_raw)
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def ensure_created(self) -> None:
+        self._ensure_sqlite_path()
+        Base.metadata.create_all(self.engine)
+
+    def _session(self) -> Session:
+        return self.session_factory()
+
+    def start_refresh_run(
+        self,
+        *,
+        mode: str,
+        requested_entity_sets: list[str],
+        date_from: str | None,
+        date_to: str | None,
+        limit_per_set: int,
+    ) -> str:
+        run_id = uuid4().hex
+        with self._session() as session, session.begin():
+            session.add(
+                RefreshRunRow(
+                    id=run_id,
+                    mode=mode,
+                    status="running",
+                    started_at=_utc_now(),
+                    requested_entity_sets_json=_dump_json(requested_entity_sets),
+                    date_from=date_from,
+                    date_to=date_to,
+                    limit_per_set=limit_per_set,
+                )
+            )
+        return run_id
+
+    def finish_refresh_run(
+        self,
+        *,
+        run_id: str,
+        status: str,
+        records_read: int,
+        entities_written: int,
+        links_written: int,
+        checkpoints_updated: int,
+        details: dict[str, Any] | None = None,
+        error_message: str | None = None,
+    ) -> None:
+        with self._session() as session, session.begin():
+            run = session.get(RefreshRunRow, run_id)
+            if run is None:
+                return
+            run.status = status
+            run.records_read = records_read
+            run.entities_written = entities_written
+            run.links_written = links_written
+            run.checkpoints_updated = checkpoints_updated
+            run.details_json = _dump_json(details or {})
+            run.error_message = error_message
+            run.finished_at = _utc_now()
+
+    def upsert_entities(self, *, run_id: str, entities: list[CanonicalEntity]) -> tuple[int, int]:
+        entities_written = 0
+        links_written = 0
+        now = _utc_now()
+
+        with self._session() as session, session.begin():
+            for entity in entities:
+                row = session.execute(
+                    select(CanonicalEntityRow).where(
+                        CanonicalEntityRow.source_entity == entity.source_entity,
+                        CanonicalEntityRow.source_id == entity.source_id,
+                    )
+                ).scalar_one_or_none()
+
+                if row is None:
+                    row = CanonicalEntityRow(
+                        source_entity=entity.source_entity,
+                        source_id=entity.source_id,
+                        display_name=entity.display_name,
+                        attributes_json=_dump_json(entity.attributes),
+                        first_seen_at=now,
+                        updated_at=now,
+                        last_refresh_run_id=run_id,
+                    )
+                    session.add(row)
+                else:
+                    row.display_name = entity.display_name
+                    row.attributes_json = _dump_json(entity.attributes)
+                    row.updated_at = now
+                    row.last_refresh_run_id = run_id
+
+                entities_written += 1
+
+                session.execute(
+                    delete(CanonicalLinkRow).where(
+                        CanonicalLinkRow.source_entity == entity.source_entity,
+                        CanonicalLinkRow.source_id == entity.source_id,
+                    )
+                )
+
+                for link in entity.links:
+                    if not link.target_id:
+                        continue
+                    session.add(
+                        CanonicalLinkRow(
+                            source_entity=entity.source_entity,
+                            source_id=entity.source_id,
+                            relation=link.relation,
+                            target_entity=link.target_entity,
+                            target_id=link.target_id,
+                            source_field=link.source_field,
+                            updated_at=now,
+                            last_refresh_run_id=run_id,
+                        )
+                    )
+                    links_written += 1
+
+        return entities_written, links_written
+
+    def update_checkpoints(
+        self,
+        *,
+        run_id: str,
+        entity_sets: list[str],
+        date_from: str | None,
+        date_to: str | None,
+    ) -> int:
+        if not entity_sets:
+            return 0
+
+        now = _utc_now()
+        updated = 0
+        with self._session() as session, session.begin():
+            for entity_set in entity_sets:
+                row = session.get(RefreshCheckpointRow, entity_set)
+                if row is None:
+                    row = RefreshCheckpointRow(
+                        entity_set=entity_set,
+                        last_success_at=now,
+                        last_refresh_run_id=run_id,
+                        last_date_from=date_from,
+                        last_date_to=date_to,
+                    )
+                    session.add(row)
+                else:
+                    row.last_success_at = now
+                    row.last_refresh_run_id = run_id
+                    row.last_date_from = date_from
+                    row.last_date_to = date_to
+                updated += 1
+        return updated
+
+    def list_recent_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 200))
+        with self._session() as session:
+            rows = (
+                session.execute(
+                    select(RefreshRunRow)
+                    .order_by(RefreshRunRow.started_at.desc())
+                    .limit(safe_limit)
+                )
+                .scalars()
+                .all()
+            )
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "run_id": row.id,
+                    "mode": row.mode,
+                    "status": row.status,
+                    "started_at": _dt_to_iso(row.started_at),
+                    "finished_at": _dt_to_iso(row.finished_at),
+                    "requested_entity_sets": _load_json(row.requested_entity_sets_json, []),
+                    "date_from": row.date_from,
+                    "date_to": row.date_to,
+                    "limit_per_set": row.limit_per_set,
+                    "records_read": row.records_read,
+                    "entities_written": row.entities_written,
+                    "links_written": row.links_written,
+                    "checkpoints_updated": row.checkpoints_updated,
+                    "details": _load_json(row.details_json, {}),
+                    "error_message": row.error_message,
+                }
+            )
+        return output
+
+    def store_stats(self) -> dict[str, Any]:
+        with self._session() as session:
+            entities_total = session.execute(select(func.count(CanonicalEntityRow.id))).scalar_one()
+            links_total = session.execute(select(func.count(CanonicalLinkRow.id))).scalar_one()
+            checkpoints_total = session.execute(select(func.count(RefreshCheckpointRow.entity_set))).scalar_one()
+            latest_run = (
+                session.execute(select(RefreshRunRow).order_by(RefreshRunRow.started_at.desc()).limit(1))
+                .scalars()
+                .first()
+            )
+
+        latest_run_payload: dict[str, Any] | None = None
+        if latest_run is not None:
+            latest_run_payload = {
+                "run_id": latest_run.id,
+                "mode": latest_run.mode,
+                "status": latest_run.status,
+                "started_at": _dt_to_iso(latest_run.started_at),
+                "finished_at": _dt_to_iso(latest_run.finished_at),
+            }
+
+        return {
+            "db_url": self.db_url,
+            "entities_total": int(entities_total),
+            "links_total": int(links_total),
+            "checkpoints_total": int(checkpoints_total),
+            "latest_run": latest_run_payload,
+        }
+
+    def start_feature_run(self, *, baseline_window_hours: int) -> str:
+        run_id = uuid4().hex
+        with self._session() as session, session.begin():
+            session.add(
+                FeatureRunRow(
+                    id=run_id,
+                    status="running",
+                    started_at=_utc_now(),
+                    baseline_window_hours=baseline_window_hours,
+                )
+            )
+        return run_id
+
+    def replace_feature_results(
+        self,
+        *,
+        run_id: str,
+        metrics: list[dict[str, Any]],
+        anomalies: list[dict[str, Any]],
+    ) -> tuple[int, int]:
+        now = _utc_now()
+        with self._session() as session, session.begin():
+            session.execute(delete(FeatureMetricRow).where(FeatureMetricRow.feature_run_id == run_id))
+            session.execute(delete(AnomalySignalRow).where(AnomalySignalRow.feature_run_id == run_id))
+
+            # Deactivate previously active anomalies before writing a new active snapshot.
+            previous_anomalies = session.execute(
+                select(AnomalySignalRow).where(AnomalySignalRow.is_active == 1)
+            ).scalars().all()
+            for item in previous_anomalies:
+                item.is_active = 0
+
+            for metric in metrics:
+                session.add(
+                    FeatureMetricRow(
+                        feature_run_id=run_id,
+                        metric_key=str(metric.get("metric_key", "")),
+                        scope=str(metric.get("scope", "global")),
+                        scope_id=str(metric.get("scope_id", "")),
+                        metric_type=str(metric.get("metric_type", "gauge")),
+                        metric_value=float(metric.get("metric_value", 0.0)),
+                        attributes_json=_dump_json(metric.get("attributes", {})),
+                        computed_at=now,
+                    )
+                )
+
+            for anomaly in anomalies:
+                session.add(
+                    AnomalySignalRow(
+                        feature_run_id=run_id,
+                        signal_type=str(anomaly.get("signal_type", "unknown_signal")),
+                        severity=str(anomaly.get("severity", "medium")),
+                        scope=str(anomaly.get("scope", "global")),
+                        scope_id=str(anomaly.get("scope_id", "")),
+                        score=float(anomaly.get("score", 0.0)),
+                        details_json=_dump_json(anomaly.get("details", {})),
+                        detected_at=now,
+                        is_active=1,
+                    )
+                )
+
+        return len(metrics), len(anomalies)
+
+    def finish_feature_run(
+        self,
+        *,
+        run_id: str,
+        status: str,
+        entities_total: int,
+        metrics_written: int,
+        anomalies_written: int,
+        details: dict[str, Any] | None = None,
+        error_message: str | None = None,
+    ) -> None:
+        with self._session() as session, session.begin():
+            row = session.get(FeatureRunRow, run_id)
+            if row is None:
+                return
+            row.status = status
+            row.entities_total = entities_total
+            row.metrics_written = metrics_written
+            row.anomalies_written = anomalies_written
+            row.details_json = _dump_json(details or {})
+            row.error_message = error_message
+            row.finished_at = _utc_now()
+
+    def list_recent_feature_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 200))
+        with self._session() as session:
+            rows = (
+                session.execute(
+                    select(FeatureRunRow)
+                    .order_by(FeatureRunRow.started_at.desc())
+                    .limit(safe_limit)
+                )
+                .scalars()
+                .all()
+            )
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "run_id": row.id,
+                    "status": row.status,
+                    "started_at": _dt_to_iso(row.started_at),
+                    "finished_at": _dt_to_iso(row.finished_at),
+                    "baseline_window_hours": row.baseline_window_hours,
+                    "entities_total": row.entities_total,
+                    "metrics_written": row.metrics_written,
+                    "anomalies_written": row.anomalies_written,
+                    "details": _load_json(row.details_json, {}),
+                    "error_message": row.error_message,
+                }
+            )
+        return output
+
+    def list_feature_metrics(
+        self,
+        *,
+        limit: int = 200,
+        metric_key: str | None = None,
+        scope: str | None = None,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 2000))
+        with self._session() as session:
+            stmt = select(FeatureMetricRow).order_by(FeatureMetricRow.computed_at.desc(), FeatureMetricRow.id.desc())
+            if metric_key:
+                stmt = stmt.where(FeatureMetricRow.metric_key == metric_key)
+            if scope:
+                stmt = stmt.where(FeatureMetricRow.scope == scope)
+            if run_id:
+                stmt = stmt.where(FeatureMetricRow.feature_run_id == run_id)
+            rows = session.execute(stmt.limit(safe_limit)).scalars().all()
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "id": row.id,
+                    "feature_run_id": row.feature_run_id,
+                    "metric_key": row.metric_key,
+                    "scope": row.scope,
+                    "scope_id": row.scope_id,
+                    "metric_type": row.metric_type,
+                    "metric_value": row.metric_value,
+                    "attributes": _load_json(row.attributes_json, {}),
+                    "computed_at": _dt_to_iso(row.computed_at),
+                }
+            )
+        return output
+
+    def list_anomaly_signals(
+        self,
+        *,
+        limit: int = 200,
+        severity: str | None = None,
+        active_only: bool = True,
+        run_id: str | None = None,
+    ) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 2000))
+        with self._session() as session:
+            stmt = select(AnomalySignalRow).order_by(AnomalySignalRow.detected_at.desc(), AnomalySignalRow.id.desc())
+            if active_only:
+                stmt = stmt.where(AnomalySignalRow.is_active == 1)
+            if severity:
+                stmt = stmt.where(AnomalySignalRow.severity == severity)
+            if run_id:
+                stmt = stmt.where(AnomalySignalRow.feature_run_id == run_id)
+            rows = session.execute(stmt.limit(safe_limit)).scalars().all()
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "id": row.id,
+                    "feature_run_id": row.feature_run_id,
+                    "signal_type": row.signal_type,
+                    "severity": row.severity,
+                    "scope": row.scope,
+                    "scope_id": row.scope_id,
+                    "score": row.score,
+                    "details": _load_json(row.details_json, {}),
+                    "detected_at": _dt_to_iso(row.detected_at),
+                    "is_active": bool(row.is_active),
+                }
+            )
+        return output
+
+    def feature_store_stats(self) -> dict[str, Any]:
+        with self._session() as session:
+            metrics_total = session.execute(select(func.count(FeatureMetricRow.id))).scalar_one()
+            anomalies_total = session.execute(select(func.count(AnomalySignalRow.id))).scalar_one()
+            active_anomalies_total = session.execute(
+                select(func.count(AnomalySignalRow.id)).where(AnomalySignalRow.is_active == 1)
+            ).scalar_one()
+            latest_feature_run = (
+                session.execute(select(FeatureRunRow).order_by(FeatureRunRow.started_at.desc()).limit(1))
+                .scalars()
+                .first()
+            )
+
+        latest_payload: dict[str, Any] | None = None
+        if latest_feature_run is not None:
+            latest_payload = {
+                "run_id": latest_feature_run.id,
+                "status": latest_feature_run.status,
+                "started_at": _dt_to_iso(latest_feature_run.started_at),
+                "finished_at": _dt_to_iso(latest_feature_run.finished_at),
+                "metrics_written": latest_feature_run.metrics_written,
+                "anomalies_written": latest_feature_run.anomalies_written,
+            }
+
+        return {
+            "metrics_total": int(metrics_total),
+            "anomalies_total": int(anomalies_total),
+            "active_anomalies_total": int(active_anomalies_total),
+            "latest_feature_run": latest_payload,
+        }
+
+    def latest_successful_feature_run(self) -> dict[str, Any] | None:
+        with self._session() as session:
+            row = (
+                session.execute(
+                    select(FeatureRunRow)
+                    .where(FeatureRunRow.status == "success")
+                    .order_by(FeatureRunRow.finished_at.desc(), FeatureRunRow.started_at.desc())
+                    .limit(1)
+                )
+                .scalars()
+                .first()
+            )
+        if row is None:
+            return None
+        return {
+            "run_id": row.id,
+            "status": row.status,
+            "started_at": _dt_to_iso(row.started_at),
+            "finished_at": _dt_to_iso(row.finished_at),
+            "metrics_written": row.metrics_written,
+            "anomalies_written": row.anomalies_written,
+        }
+
+    def latest_refresh_finished_at(self) -> datetime | None:
+        with self._session() as session:
+            row = (
+                session.execute(
+                    select(RefreshRunRow)
+                    .where(RefreshRunRow.status.in_(("success", "partial_success")))
+                    .order_by(RefreshRunRow.finished_at.desc())
+                    .limit(1)
+                )
+                .scalars()
+                .first()
+            )
+        if row is None:
+            return None
+        return row.finished_at
+
+    def iter_entities_for_features(self, *, limit: int = 200000) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 200000))
+        with self._session() as session:
+            rows = (
+                session.execute(
+                    select(CanonicalEntityRow)
+                    .order_by(CanonicalEntityRow.updated_at.desc())
+                    .limit(safe_limit)
+                )
+                .scalars()
+                .all()
+            )
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "source_entity": row.source_entity,
+                    "source_id": row.source_id,
+                    "display_name": row.display_name,
+                    "attributes": _load_json(row.attributes_json, {}),
+                    "updated_at": row.updated_at,
+                }
+            )
+        return output
+
+    def link_counts_by_source(self) -> dict[tuple[str, str], int]:
+        with self._session() as session:
+            rows = session.execute(
+                select(
+                    CanonicalLinkRow.source_entity,
+                    CanonicalLinkRow.source_id,
+                    func.count(CanonicalLinkRow.id),
+                )
+                .group_by(CanonicalLinkRow.source_entity, CanonicalLinkRow.source_id)
+            ).all()
+
+        output: dict[tuple[str, str], int] = {}
+        for source_entity, source_id, count in rows:
+            output[(str(source_entity), str(source_id))] = int(count)
+        return output
+
+    def start_risk_run(self, *, source_feature_run_id: str | None) -> str:
+        run_id = uuid4().hex
+        with self._session() as session, session.begin():
+            session.add(
+                RiskRunRow(
+                    id=run_id,
+                    status="running",
+                    started_at=_utc_now(),
+                    source_feature_run_id=source_feature_run_id,
+                )
+            )
+        return run_id
+
+    def replace_risk_patterns(self, *, run_id: str, patterns: list[dict[str, Any]]) -> int:
+        now = _utc_now()
+        with self._session() as session, session.begin():
+            session.execute(delete(RiskPatternRow).where(RiskPatternRow.risk_run_id == run_id))
+
+            previous_active = session.execute(
+                select(RiskPatternRow).where(RiskPatternRow.is_active == 1)
+            ).scalars().all()
+            for item in previous_active:
+                item.is_active = 0
+
+            for pattern in patterns:
+                session.add(
+                    RiskPatternRow(
+                        risk_run_id=run_id,
+                        pattern_key=str(pattern.get("pattern_key", "unknown_pattern")),
+                        severity=str(pattern.get("severity", "low")),
+                        scope=str(pattern.get("scope", "global")),
+                        scope_id=str(pattern.get("scope_id", "")),
+                        score=float(pattern.get("score", 0.0)),
+                        confidence=float(pattern.get("confidence", 0.0)),
+                        details_json=_dump_json(pattern.get("details", {})),
+                        detected_at=now,
+                        is_active=1,
+                    )
+                )
+        return len(patterns)
+
+    def finish_risk_run(
+        self,
+        *,
+        run_id: str,
+        status: str,
+        patterns_written: int,
+        global_score: float,
+        details: dict[str, Any] | None = None,
+        error_message: str | None = None,
+    ) -> None:
+        with self._session() as session, session.begin():
+            row = session.get(RiskRunRow, run_id)
+            if row is None:
+                return
+            row.status = status
+            row.patterns_written = patterns_written
+            row.global_score = global_score
+            row.details_json = _dump_json(details or {})
+            row.error_message = error_message
+            row.finished_at = _utc_now()
+
+    def list_recent_risk_runs(self, limit: int = 20) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 200))
+        with self._session() as session:
+            rows = (
+                session.execute(
+                    select(RiskRunRow)
+                    .order_by(RiskRunRow.started_at.desc())
+                    .limit(safe_limit)
+                )
+                .scalars()
+                .all()
+            )
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "run_id": row.id,
+                    "status": row.status,
+                    "started_at": _dt_to_iso(row.started_at),
+                    "finished_at": _dt_to_iso(row.finished_at),
+                    "source_feature_run_id": row.source_feature_run_id,
+                    "patterns_written": row.patterns_written,
+                    "global_score": row.global_score,
+                    "details": _load_json(row.details_json, {}),
+                    "error_message": row.error_message,
+                }
+            )
+        return output
+
+    def list_risk_patterns(
+        self,
+        *,
+        limit: int = 200,
+        severity: str | None = None,
+        active_only: bool = True,
+        run_id: str | None = None,
+        pattern_key: str | None = None,
+        scope: str | None = None,
+    ) -> list[dict[str, Any]]:
+        safe_limit = max(1, min(limit, 2000))
+        with self._session() as session:
+            stmt = select(RiskPatternRow).order_by(RiskPatternRow.detected_at.desc(), RiskPatternRow.id.desc())
+            if active_only:
+                stmt = stmt.where(RiskPatternRow.is_active == 1)
+            if severity:
+                stmt = stmt.where(RiskPatternRow.severity == severity)
+            if run_id:
+                stmt = stmt.where(RiskPatternRow.risk_run_id == run_id)
+            if pattern_key:
+                stmt = stmt.where(RiskPatternRow.pattern_key == pattern_key)
+            if scope:
+                stmt = stmt.where(RiskPatternRow.scope == scope)
+            rows = session.execute(stmt.limit(safe_limit)).scalars().all()
+
+        output: list[dict[str, Any]] = []
+        for row in rows:
+            output.append(
+                {
+                    "id": row.id,
+                    "risk_run_id": row.risk_run_id,
+                    "pattern_key": row.pattern_key,
+                    "severity": row.severity,
+                    "scope": row.scope,
+                    "scope_id": row.scope_id,
+                    "score": row.score,
+                    "confidence": row.confidence,
+                    "details": _load_json(row.details_json, {}),
+                    "detected_at": _dt_to_iso(row.detected_at),
+                    "is_active": bool(row.is_active),
+                }
+            )
+        return output
+
+    def risk_store_stats(self) -> dict[str, Any]:
+        with self._session() as session:
+            patterns_total = session.execute(select(func.count(RiskPatternRow.id))).scalar_one()
+            active_patterns_total = session.execute(
+                select(func.count(RiskPatternRow.id)).where(RiskPatternRow.is_active == 1)
+            ).scalar_one()
+            latest_run = (
+                session.execute(select(RiskRunRow).order_by(RiskRunRow.started_at.desc()).limit(1))
+                .scalars()
+                .first()
+            )
+
+        latest_payload: dict[str, Any] | None = None
+        if latest_run is not None:
+            latest_payload = {
+                "run_id": latest_run.id,
+                "status": latest_run.status,
+                "started_at": _dt_to_iso(latest_run.started_at),
+                "finished_at": _dt_to_iso(latest_run.finished_at),
+                "patterns_written": latest_run.patterns_written,
+                "global_score": latest_run.global_score,
+            }
+
+        return {
+            "patterns_total": int(patterns_total),
+            "active_patterns_total": int(active_patterns_total),
+            "latest_risk_run": latest_payload,
+        }

canonical_layer/store_models.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import DateTime, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+def utc_now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class RefreshRunRow(Base):
+    __tablename__ = "refresh_runs"
+
+    id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    mode: Mapped[str] = mapped_column(String(32), nullable=False)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="running")
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    requested_entity_sets_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
+    date_from: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    date_to: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    limit_per_set: Mapped[int] = mapped_column(Integer, nullable=False, default=200)
+    records_read: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    entities_written: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    links_written: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    checkpoints_updated: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class CanonicalEntityRow(Base):
+    __tablename__ = "canonical_entities"
+    __table_args__ = (
+        UniqueConstraint("source_entity", "source_id", name="uq_canonical_entities_source"),
+    )
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    source_entity: Mapped[str] = mapped_column(String(255), nullable=False)
+    source_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    display_name: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    attributes_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    last_refresh_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+
+
+class CanonicalLinkRow(Base):
+    __tablename__ = "canonical_links"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    source_entity: Mapped[str] = mapped_column(String(255), nullable=False)
+    source_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    relation: Mapped[str] = mapped_column(String(128), nullable=False)
+    target_entity: Mapped[str] = mapped_column(String(255), nullable=False)
+    target_id: Mapped[str] = mapped_column(String(255), nullable=False)
+    source_field: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    last_refresh_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+
+
+class RefreshCheckpointRow(Base):
+    __tablename__ = "refresh_checkpoints"
+
+    entity_set: Mapped[str] = mapped_column(String(255), primary_key=True)
+    last_success_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    last_refresh_run_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    last_date_from: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    last_date_to: Mapped[str | None] = mapped_column(String(64), nullable=True)
+
+
+class FeatureRunRow(Base):
+    __tablename__ = "feature_runs"
+
+    id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="running")
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    baseline_window_hours: Mapped[int] = mapped_column(Integer, nullable=False, default=24)
+    entities_total: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    metrics_written: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    anomalies_written: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class FeatureMetricRow(Base):
+    __tablename__ = "feature_metrics"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    feature_run_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    metric_key: Mapped[str] = mapped_column(String(128), nullable=False)
+    scope: Mapped[str] = mapped_column(String(128), nullable=False)
+    scope_id: Mapped[str] = mapped_column(String(255), nullable=False, default="")
+    metric_type: Mapped[str] = mapped_column(String(32), nullable=False, default="gauge")
+    metric_value: Mapped[float] = mapped_column(nullable=False, default=0.0)
+    attributes_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    computed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+
+
+class AnomalySignalRow(Base):
+    __tablename__ = "anomaly_signals"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    feature_run_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    signal_type: Mapped[str] = mapped_column(String(128), nullable=False)
+    severity: Mapped[str] = mapped_column(String(32), nullable=False, default="medium")
+    scope: Mapped[str] = mapped_column(String(128), nullable=False, default="global")
+    scope_id: Mapped[str] = mapped_column(String(255), nullable=False, default="")
+    score: Mapped[float] = mapped_column(nullable=False, default=0.0)
+    details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    is_active: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+
+
+class RiskRunRow(Base):
+    __tablename__ = "risk_runs"
+
+    id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    status: Mapped[str] = mapped_column(String(32), nullable=False, default="running")
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    source_feature_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    patterns_written: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    global_score: Mapped[float] = mapped_column(nullable=False, default=0.0)
+    details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class RiskPatternRow(Base):
+    __tablename__ = "risk_patterns"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    risk_run_id: Mapped[str] = mapped_column(String(64), nullable=False)
+    pattern_key: Mapped[str] = mapped_column(String(128), nullable=False)
+    severity: Mapped[str] = mapped_column(String(32), nullable=False, default="low")
+    scope: Mapped[str] = mapped_column(String(128), nullable=False, default="global")
+    scope_id: Mapped[str] = mapped_column(String(255), nullable=False, default="")
+    score: Mapped[float] = mapped_column(nullable=False, default=0.0)
+    confidence: Mapped[float] = mapped_column(nullable=False, default=0.0)
+    details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
+    detected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=utc_now)
+    is_active: Mapped[int] = mapped_column(Integer, nullable=False, default=1)