Initial import NDC_1C

This commit is contained in:
2026-03-26 10:38:25 +03:00
commit a162d77ef7
2943 changed files with 3615871 additions and 0 deletions
Binary file not shown.
+49
View File
@@ -0,0 +1,49 @@
from __future__ import annotations
from orchestration.batch_runtime import enqueue_refresh_and_answer_job, run_refresh_and_answer_job
def test_batch_runtime_handoff_executes_feature_and_risk() -> None:
job = enqueue_refresh_and_answer_job(
question_id="Q28",
slice_window="2020-06",
requested_outputs=["feature_store", "risk_store"],
reason=["needs_ranking", "aggregate_not_sufficient"],
)
result = run_refresh_and_answer_job(
job,
feature_executor=lambda: {"run_id": "feature-run-1"},
risk_executor=lambda: {"run_id": "risk-run-1"},
should_refresh=False,
)
payload = result.to_dict()
assert payload["status"] == "success"
assert payload["execution_mode"] == "batch_runtime_executed"
assert payload["run_ids"]["feature_run_id"] == "feature-run-1"
assert payload["run_ids"]["risk_run_id"] == "risk-run-1"
def test_batch_runtime_handoff_can_execute_refresh() -> None:
job = enqueue_refresh_and_answer_job(
question_id="Q30",
slice_window="2020-06",
requested_outputs=["feature_store", "risk_store"],
reason=["needs_full_period_aggregation", "refresh_stale"],
)
result = run_refresh_and_answer_job(
job,
refresh_executor=lambda: {"run_id": "refresh-run-1"},
feature_executor=lambda: {"run_id": "feature-run-2"},
risk_executor=lambda: {"run_id": "risk-run-2"},
should_refresh=True,
)
payload = result.to_dict()
assert payload["status"] == "success"
assert payload["run_ids"]["refresh_run_id"] == "refresh-run-1"
assert payload["run_ids"]["feature_run_id"] == "feature-run-2"
assert payload["run_ids"]["risk_run_id"] == "risk-run-2"
+112
View File
@@ -0,0 +1,112 @@
from __future__ import annotations
from pathlib import Path
from canonical_layer.features import FeatureService
from canonical_layer.models import CanonicalEntity, EntityLink
from canonical_layer.store import CanonicalStore
from config.settings import OneCSettings
def _build_settings(db_url: str) -> OneCSettings:
return OneCSettings(
base_url="http://localhost",
infobase="buh_test",
username="",
password="",
odata_path="/odata/standard.odata/",
timeout=30,
verify_tls=False,
probe_top=5,
probe_entity_sets=(),
canonical_db_url=db_url,
refresh_default_limit_per_set=50,
refresh_default_entity_keywords=("document", "posting"),
feature_default_baseline_window_hours=24,
anomaly_stale_refresh_threshold_hours=6,
feature_entity_scan_limit=200000,
risk_medium_threshold=0.45,
risk_high_threshold=0.75,
risk_anomaly_scan_limit=5000,
)
def _seed_entities(store: CanonicalStore) -> None:
links = [
EntityLink(
relation="reference",
target_entity="Counterparty",
target_id=f"cp-{idx}",
source_field="Counterparty_Key",
)
for idx in range(12)
]
entities = [
CanonicalEntity(
source_entity="DocumentSales",
source_id="doc-001",
display_name="Invoice 001",
attributes={"Amount": 1000},
links=links,
),
CanonicalEntity(
source_entity="DocumentSales",
source_id="doc-002",
display_name="Invoice 002",
attributes={"Amount": 500},
links=[],
),
]
store.upsert_entities(run_id="seed", entities=entities)
def test_feature_engine_generates_metrics_and_anomalies(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'feature_engine.db').as_posix()}"
settings = _build_settings(db_url)
store = CanonicalStore(settings.canonical_db_url)
store.ensure_created()
_seed_entities(store)
refresh_run_id = store.start_refresh_run(
mode="incremental",
requested_entity_sets=["DocumentSales"],
date_from=None,
date_to=None,
limit_per_set=10,
)
store.finish_refresh_run(
run_id=refresh_run_id,
status="success",
records_read=2,
entities_written=2,
links_written=12,
checkpoints_updated=1,
details={},
)
service = FeatureService(settings=settings, store=store)
result = service.run_feature_engine(top_account_tokens=10)
assert result.status == "success"
assert result.metrics_written > 0
assert result.anomalies_written > 0
anomalies = service.list_anomalies(limit=100, active_only=True)
signal_types = {item["signal_type"] for item in anomalies}
assert "high_link_degree" in signal_types
def test_feature_engine_detects_missing_refresh_baseline(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'feature_engine_no_refresh.db').as_posix()}"
settings = _build_settings(db_url)
store = CanonicalStore(settings.canonical_db_url)
store.ensure_created()
_seed_entities(store)
service = FeatureService(settings=settings, store=store)
result = service.run_feature_engine(top_account_tokens=10)
assert result.status == "success"
anomalies = service.list_anomalies(limit=100, active_only=True)
signal_types = {item["signal_type"] for item in anomalies}
assert "missing_refresh_baseline" in signal_types
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
from canonical_layer.mappers import map_record
def _find_link(entity, source_field: str):
for link in entity.links:
if link.source_field == source_field:
return link
return None
def test_map_record_keeps_explicit_identity_and_links() -> None:
row = {
"Ref_Key": "11111111-2222-3333-4444-555555555555",
"Description": "Документ тест",
"Counterparty_Key": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
}
entity = map_record("DocumentSales", row)
assert entity.source_id == "11111111-2222-3333-4444-555555555555"
assert entity.display_name == "Документ тест"
link = _find_link(entity, "Counterparty_Key")
assert link is not None
assert link.target_entity == "Counterparty"
assert link.relation == "document_has_counterparty"
def test_map_record_builds_composite_source_id_for_registers() -> None:
row = {
"Recorder": "12345678-1111-2222-3333-123456789abc",
"Recorder_Type": "StandardODATA.Document_РеализацияТоваровУслуг",
"LineNumber": "7",
"Period": "2020-06-01T00:00:00",
}
entity = map_record("AccumulationRegister_НДСЗаписиКнигиПродаж_RecordType", row)
assert entity.source_id.startswith("cmp:")
recorder_link = _find_link(entity, "Recorder")
assert recorder_link is not None
assert recorder_link.target_entity == "Document"
assert recorder_link.relation == "register_recorded_by_document"
def test_map_record_journal_ref_points_to_document() -> None:
row = {
"Ref": "22222222-3333-4444-5555-666666666666",
"Ref_Type": "StandardODATA.Document_СписаниеСРасчетногоСчета",
"Description": "Журнал банковских выписок",
}
entity = map_record("DocumentJournal_БанковскиеВыписки", row)
assert entity.source_id == "22222222-3333-4444-5555-666666666666"
ref_link = _find_link(entity, "Ref")
assert ref_link is not None
assert ref_link.target_entity == "Document"
assert ref_link.relation == "journal_refers_to_document"
def test_map_record_supplier_and_buyer_are_typed_counterparties() -> None:
row = {
"Recorder": "44444444-1111-2222-3333-abcdefabcdef",
"Recorder_Type": "StandardODATA.Document_ПоступлениеТоваровУслуг",
"Поставщик_Key": "aaaaaaaa-1111-2222-3333-bbbbbbbbbbbb",
"Покупатель_Key": "cccccccc-1111-2222-3333-dddddddddddd",
}
entity = map_record("AccumulationRegister_НДСПредъявленный_RecordType", row)
supplier = _find_link(entity, "Поставщик_Key")
buyer = _find_link(entity, "Покупатель_Key")
assert supplier is not None
assert buyer is not None
assert supplier.target_entity == "Counterparty"
assert buyer.target_entity == "Counterparty"
assert supplier.relation == "register_relates_to_supplier"
assert buyer.relation == "register_relates_to_buyer"
def test_map_record_invoice_not_misclassified_as_account() -> None:
row = {
"Recorder": "99999999-1111-2222-3333-eeeeeeeeeeee",
"Recorder_Type": "StandardODATA.Document_ФормированиеЗаписейКнигиПокупок",
"СчетФактура": "11111111-aaaa-bbbb-cccc-222222222222",
"СчетФактура_Type": "StandardODATA.Document_СчетФактураПолученный",
}
entity = map_record("AccumulationRegister_НДСЗаписиКнигиПокупок_RecordType", row)
invoice = _find_link(entity, "СчетФактура")
assert invoice is not None
assert invoice.target_entity == "InvoiceDocument"
assert invoice.relation == "register_relates_to_invoice"
def test_map_record_zero_guid_links_are_filtered_out() -> None:
row = {
"Ref_Key": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"Counterparty_Key": "00000000-0000-0000-0000-000000000000",
}
entity = map_record("Document_РеализацияТоваровУслуг", row)
assert _find_link(entity, "Counterparty_Key") is None
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
from datetime import datetime, timezone
from canonical_layer.period_snapshot import (
normalize_dt,
parse_dt,
parse_record_datetime,
window_bounds_from_key,
window_key,
)
def test_parse_dt_odata_and_iso() -> None:
from_odata = parse_dt("/Date(1583020800000)/")
assert from_odata is not None
assert from_odata.year == 2020
assert from_odata.month == 3
assert from_odata.day == 1
from_iso = parse_dt("2020-03-01T00:00:00Z")
assert from_iso is not None
assert from_iso.tzinfo is not None
assert normalize_dt(from_iso).year == 2020
def test_window_key_and_bounds_month() -> None:
dt = datetime(2020, 11, 15, 12, 0, 0, tzinfo=timezone.utc)
key = window_key(dt, granularity="month")
assert key == "2020-11"
start, end = window_bounds_from_key(key, granularity="month")
assert start.isoformat() == "2020-11-01T00:00:00+00:00"
assert end.isoformat() == "2020-12-01T00:00:00+00:00"
def test_window_key_and_bounds_week() -> None:
dt = datetime(2020, 12, 30, 7, 0, 0, tzinfo=timezone.utc)
key = window_key(dt, granularity="week")
assert key.startswith("2020-W")
start, end = window_bounds_from_key(key, granularity="week")
assert end > start
def test_parse_record_datetime_finds_common_fields() -> None:
row = {"Date": "2020-05-20T10:00:00Z"}
dt = parse_record_datetime(row)
assert dt is not None
assert dt.year == 2020
assert dt.month == 5
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
from pathlib import Path
from canonical_layer.refresh import RefreshService
from canonical_layer.store import CanonicalStore
from config.settings import OneCSettings
class FakeClient:
def __init__(self, payload_by_set: dict[str, object]) -> None:
self.payload_by_set = payload_by_set
def read_entity_set_records(self, entity_set: str, top: int = 5, extra_params: dict | None = None) -> list[dict]:
payload = self.payload_by_set.get(entity_set, [])
if isinstance(payload, Exception):
raise payload
assert isinstance(payload, list)
return payload[:top]
def fetch_metadata(self) -> str:
return "<edmx:Edmx xmlns:edmx='http://docs.oasis-open.org/odata/ns/edmx'/>"
def _build_settings(db_url: str) -> OneCSettings:
return OneCSettings(
base_url="http://localhost",
infobase="buh_test",
username="",
password="",
odata_path="/odata/standard.odata/",
timeout=30,
verify_tls=False,
probe_top=5,
probe_entity_sets=(),
canonical_db_url=db_url,
refresh_default_limit_per_set=50,
refresh_default_entity_keywords=("document", "posting"),
feature_default_baseline_window_hours=24,
anomaly_stale_refresh_threshold_hours=6,
feature_entity_scan_limit=200000,
risk_medium_threshold=0.45,
risk_high_threshold=0.75,
risk_anomaly_scan_limit=5000,
)
def test_refresh_writes_entities_and_links(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'canonical.db').as_posix()}"
settings = _build_settings(db_url)
client = FakeClient(
{
"DocumentSales": [
{
"Ref_Key": "11111111-2222-3333-4444-555555555555",
"Description": "Doc 1",
"Counterparty_Key": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
},
{
"Ref_Key": "66666666-7777-8888-9999-000000000000",
"Description": "Doc 2",
},
],
}
)
store = CanonicalStore(settings.canonical_db_url)
service = RefreshService(settings=settings, client=client, store=store)
result = service.run_refresh(
mode="historical",
requested_entity_sets=["DocumentSales"],
limit_per_set=100,
)
assert result.status == "success"
assert result.records_read == 2
assert result.entities_written == 2
assert result.links_written >= 1
stats = service.store_stats()
assert stats["entities_total"] == 2
assert stats["checkpoints_total"] == 1
runs = service.list_recent_runs(limit=5)
assert runs
assert runs[0]["status"] == "success"
def test_refresh_partial_success_when_one_set_fails(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'canonical_partial.db').as_posix()}"
settings = _build_settings(db_url)
client = FakeClient(
{
"DocumentSales": [
{"Ref_Key": "11111111-2222-3333-4444-555555555555", "Description": "Doc 1"},
],
"BrokenSet": RuntimeError("boom"),
}
)
store = CanonicalStore(settings.canonical_db_url)
service = RefreshService(settings=settings, client=client, store=store)
result = service.run_refresh(
mode="incremental",
requested_entity_sets=["DocumentSales", "BrokenSet"],
limit_per_set=100,
)
assert result.status == "partial_success"
assert result.successful_entity_sets == ["DocumentSales"]
assert result.failed_entity_sets
+95
View File
@@ -0,0 +1,95 @@
from __future__ import annotations
from pathlib import Path
from canonical_layer.risk import RiskService
from canonical_layer.store import CanonicalStore
from config.settings import OneCSettings
def _build_settings(db_url: str) -> OneCSettings:
return OneCSettings(
base_url="http://localhost",
infobase="buh_test",
username="",
password="",
odata_path="/odata/standard.odata/",
timeout=30,
verify_tls=False,
probe_top=5,
probe_entity_sets=(),
canonical_db_url=db_url,
refresh_default_limit_per_set=50,
refresh_default_entity_keywords=("document", "posting"),
feature_default_baseline_window_hours=24,
anomaly_stale_refresh_threshold_hours=6,
feature_entity_scan_limit=200000,
risk_medium_threshold=0.45,
risk_high_threshold=0.75,
risk_anomaly_scan_limit=5000,
)
def test_risk_engine_builds_patterns_from_feature_anomalies(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'risk_engine.db').as_posix()}"
settings = _build_settings(db_url)
store = CanonicalStore(settings.canonical_db_url)
store.ensure_created()
feature_run_id = store.start_feature_run(baseline_window_hours=24)
store.replace_feature_results(
run_id=feature_run_id,
metrics=[],
anomalies=[
{
"signal_type": "high_link_degree",
"severity": "high",
"scope": "DocumentSales",
"scope_id": "doc-001",
"score": 1.8,
"details": {"link_count": 50},
},
{
"signal_type": "entity_count_drift",
"severity": "medium",
"scope": "source_entity",
"scope_id": "DocumentSales",
"score": 0.4,
"details": {"drift_ratio": 0.4},
},
],
)
store.finish_feature_run(
run_id=feature_run_id,
status="success",
entities_total=10,
metrics_written=0,
anomalies_written=2,
details={},
)
service = RiskService(settings=settings, store=store)
result = service.run_risk_engine()
assert result.status == "success"
assert result.patterns_written >= 3
patterns = service.list_patterns(limit=100, active_only=True)
pattern_keys = {item["pattern_key"] for item in patterns}
assert "global_risk_summary" in pattern_keys
assert "suspicious_link_hub_risk" in pattern_keys
assert "structural_drift_risk" in pattern_keys
def test_risk_engine_handles_missing_feature_baseline(tmp_path: Path) -> None:
db_url = f"sqlite:///{(tmp_path / 'risk_engine_missing_feature.db').as_posix()}"
settings = _build_settings(db_url)
store = CanonicalStore(settings.canonical_db_url)
store.ensure_created()
service = RiskService(settings=settings, store=store)
result = service.run_risk_engine()
assert result.status == "success"
patterns = service.list_patterns(limit=100, active_only=True)
pattern_keys = {item["pattern_key"] for item in patterns}
assert "global_risk_summary" in pattern_keys
assert "operational_freshness_risk" in pattern_keys
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
from router.query_classifier import RouteDecisionFlags
from router.route_selector import choose_route
from router.store_sufficiency import StoreSufficiencyResult
def _flags() -> RouteDecisionFlags:
return RouteDecisionFlags(
needs_exact_object_trace=False,
needs_causal_chain=False,
needs_cross_entity_join=False,
needs_full_period_aggregation=False,
needs_ranking=False,
needs_anomaly_summary=False,
needs_runtime_truth=False,
freshness_sensitive=False,
ambiguous_object_scope=False,
store_sufficiency_confident=False,
precomputed_aggregate_available=True,
)
def _suff() -> StoreSufficiencyResult:
return StoreSufficiencyResult(
canonical_sufficient=True,
feature_sufficient=True,
risk_sufficient=True,
freshness_ok=True,
aggregate_level_ok=True,
ranking_ready=True,
explanation_ready=True,
reason_codes=[],
)
def test_route_guard_exact_object_trace_to_live() -> None:
flags = _flags()
flags.needs_exact_object_trace = True
result = choose_route(flags, _suff(), parsed_as_trend_or_risk=False)
assert result.chosen_route == "live_mcp_drilldown"
def test_route_guard_heavy_without_aggregate_to_batch() -> None:
flags = _flags()
flags.needs_full_period_aggregation = True
flags.precomputed_aggregate_available = False
suff = _suff()
suff.aggregate_level_ok = False
result = choose_route(flags, suff, parsed_as_trend_or_risk=False)
assert result.chosen_route == "batch_refresh_then_store"
def test_route_guard_cross_entity_causal_to_hybrid() -> None:
flags = _flags()
flags.needs_cross_entity_join = True
flags.needs_causal_chain = True
suff = _suff()
suff.explanation_ready = False
result = choose_route(flags, suff, parsed_as_trend_or_risk=False)
assert result.chosen_route == "hybrid_store_plus_live"
def test_route_guard_simple_factual_to_store_canonical() -> None:
result = choose_route(_flags(), _suff(), parsed_as_trend_or_risk=False)
assert result.chosen_route == "store_canonical"
def test_route_guard_trend_to_store_feature_risk() -> None:
result = choose_route(_flags(), _suff(), parsed_as_trend_or_risk=True)
assert result.chosen_route == "store_feature_risk"
+67
View File
@@ -0,0 +1,67 @@
from __future__ import annotations
from router.query_classifier import classify_query_for_route
from router.route_selector import choose_route
from router.store_sufficiency import check_store_sufficiency
def _metadata() -> dict:
return {
"freshness_threshold_hours": 6.0,
"refresh_age_hours": 1.0,
"feature_age_hours": 1.0,
"risk_age_hours": 1.0,
"feature_ready": True,
"risk_ready": True,
"ranking_ready": False,
"aggregate_ready": True,
"precomputed_aggregates": [
"baseline_period_summary",
"period_trend_summary",
],
"canonical_semantic_coverage": 0.95,
"canonical_relation_types": 25,
"canonical_links_total": 2000,
"canonical_entities_total": 400,
}
def _route(question_class: str, question_text: str) -> str:
meta = _metadata()
flags = classify_query_for_route(question_text, {"question_class": question_class}, meta)
suff = check_store_sufficiency(flags, meta)
parsed_as_trend_or_risk = question_class in {"period_trend", "anomaly_control", "ambiguous_fuzzy"} or (
question_class == "heavy_analytical" and "baseline" in question_text.lower()
)
selection = choose_route(flags, suff, parsed_as_trend_or_risk=parsed_as_trend_or_risk)
return selection.chosen_route
def test_router_subset_q06_to_q12() -> None:
cases = [
("drilldown_explain", "Объясни сальдо через движения.", "hybrid_store_plus_live"),
("drilldown_explain", "Почему проводка на этот счет?", "live_mcp_drilldown"),
("drilldown_explain", "Цепочка документ -> проводки -> субконто.", "live_mcp_drilldown"),
("drilldown_explain", "Источник регистра для строки движения.", "live_mcp_drilldown"),
("drilldown_explain", "Почему выбрано это субконто3?", "live_mcp_drilldown"),
("cross_entity", "Свяжи документы покупателей и проводки.", "hybrid_store_plus_live"),
("cross_entity", "Свяжи контрагентов, договоры и проводки.", "hybrid_store_plus_live"),
]
for question_class, text, expected_route in cases:
assert _route(question_class, text) == expected_route
def test_router_subset_q26_to_q30() -> None:
cases = [
("heavy_analytical", "Полный риск-срез за июнь.", "batch_refresh_then_store"),
("heavy_analytical", "Рейтинг риск-счетов.", "batch_refresh_then_store"),
("heavy_analytical", "Рейтинг риск-контрагентов.", "batch_refresh_then_store"),
("heavy_analytical", "Baseline closed/open periods.", "store_feature_risk"),
("heavy_analytical", "Company anomaly summary.", "batch_refresh_then_store"),
]
for question_class, text, expected_route in cases:
assert _route(question_class, text) == expected_route
def test_router_period_trend_anomaly_stays_feature_store() -> None:
assert _route("period_trend", "Аномальный рост расходных операций?") == "store_feature_risk"
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
from router.query_classifier import classify_query_for_route
def _store_meta() -> dict:
return {
"precomputed_aggregates": ["baseline_period_summary", "period_trend_summary", "risk_slice_summary"],
}
def test_classifier_heavy_ranking_flags() -> None:
flags = classify_query_for_route(
"Рейтинг риск-счетов за июнь",
{"question_class": "heavy_analytical"},
_store_meta(),
)
assert flags.needs_full_period_aggregation is True
assert flags.needs_ranking is True
assert flags.needs_exact_object_trace is False
def test_classifier_drilldown_exact_trace_flags() -> None:
flags = classify_query_for_route(
"Цепочка документ -> проводки -> субконто",
{"question_class": "drilldown_explain"},
_store_meta(),
)
assert flags.needs_exact_object_trace is True
assert flags.needs_causal_chain is True
assert flags.needs_runtime_truth is True
def test_classifier_cross_entity_causal_flags() -> None:
flags = classify_query_for_route(
"Свяжи контрагентов, договоры и проводки",
{"question_class": "cross_entity"},
_store_meta(),
)
assert flags.needs_cross_entity_join is True
assert flags.needs_causal_chain is True
assert flags.needs_exact_object_trace is False
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
from router.query_classifier import RouteDecisionFlags
from router.store_sufficiency import check_store_sufficiency
def _base_flags() -> RouteDecisionFlags:
return RouteDecisionFlags(
needs_exact_object_trace=False,
needs_causal_chain=False,
needs_cross_entity_join=False,
needs_full_period_aggregation=False,
needs_ranking=False,
needs_anomaly_summary=False,
needs_runtime_truth=False,
freshness_sensitive=False,
ambiguous_object_scope=False,
store_sufficiency_confident=True,
precomputed_aggregate_available=True,
)
def test_store_sufficiency_positive_case() -> None:
flags = _base_flags()
metadata = {
"freshness_threshold_hours": 6.0,
"refresh_age_hours": 1.0,
"feature_age_hours": 1.0,
"risk_age_hours": 1.0,
"feature_ready": True,
"risk_ready": True,
"ranking_ready": True,
"aggregate_ready": True,
"canonical_semantic_coverage": 0.95,
"canonical_relation_types": 30,
"canonical_links_total": 1000,
"canonical_entities_total": 500,
}
result = check_store_sufficiency(flags, metadata)
assert result.canonical_sufficient is True
assert result.freshness_ok is True
assert result.reason_codes == []
def test_store_sufficiency_heavy_ranking_not_ready() -> None:
flags = _base_flags()
flags.needs_full_period_aggregation = True
flags.needs_ranking = True
flags.precomputed_aggregate_available = False
flags.freshness_sensitive = True
metadata = {
"freshness_threshold_hours": 6.0,
"refresh_age_hours": 12.0,
"feature_age_hours": 10.0,
"risk_age_hours": 10.0,
"feature_ready": True,
"risk_ready": True,
"ranking_ready": False,
"aggregate_ready": True,
"canonical_semantic_coverage": 0.95,
"canonical_relation_types": 30,
"canonical_links_total": 1000,
"canonical_entities_total": 500,
}
result = check_store_sufficiency(flags, metadata)
assert result.freshness_ok is False
assert result.aggregate_level_ok is False
assert result.ranking_ready is False
assert "aggregate_not_sufficient" in result.reason_codes
assert "ranking_not_ready" in result.reason_codes
+65
View File
@@ -0,0 +1,65 @@
from __future__ import annotations
from pathlib import Path
import runpy
from canonical_layer.store import CanonicalStore
def _load_validation_module() -> dict[str, object]:
script_path = Path(__file__).resolve().parents[1] / "scripts" / "run_validation_accounting_analytics.py"
return runpy.run_path(str(script_path), run_name="validation_mod")
def test_ingestion_handles_unknown_ids_and_deduplicates(tmp_path: Path) -> None:
module = _load_validation_module()
ingest_slice_to_store = module["ingest_slice_to_store"]
store = CanonicalStore(f"sqlite:///{(tmp_path / 'validation_ingest.db').as_posix()}")
store.ensure_created()
payload = {
"selected_window_key": "2020-06",
"records_exported_total": 3,
"links_exported_total": 0,
"records_per_entity_set": {"DocumentJournal_Test": 3},
"items": [
{
"source_entity": "DocumentJournal_Test",
"source_id": "unknown",
"display_name": "A",
"attributes": {"Recorder": "rec-1", "LineNumber": 1, "Period": "2020-06-01T00:00:00"},
"links": [],
},
{
"source_entity": "DocumentJournal_Test",
"source_id": "unknown",
"display_name": "A duplicate",
"attributes": {"Recorder": "rec-1", "LineNumber": 1, "Period": "2020-06-01T00:00:00"},
"links": [],
},
{
"source_entity": "DocumentJournal_Test",
"source_id": "unknown",
"display_name": "B",
"attributes": {"Recorder": "rec-1", "LineNumber": 2, "Period": "2020-06-01T00:00:00"},
"links": [],
},
],
}
result = ingest_slice_to_store(
store=store,
slice_payload=payload,
slice_start="2020-06-01T00:00:00+00:00",
slice_end_exclusive="2020-07-01T00:00:00+00:00",
)
assert result["entities_written"] == 2
assert result["details"]["items_total_raw"] == 3
assert result["details"]["items_after_dedupe"] == 2
assert result["details"]["duplicate_rows_skipped"] == 1
assert result["details"]["synthetic_ids_assigned"] == 3
stats = store.store_stats()
assert stats["entities_total"] == 2
BIN
View File
Binary file not shown.