Initial import NDC_1C
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""OData probe scripts for 1C metadata and link discovery."""
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from config.client import ODataClient, flatten_guid_like_fields, utc_now_iso
|
||||
from config.settings import LOGS_DIR, load_settings
|
||||
|
||||
|
||||
def _load_probe_targets() -> list[str]:
|
||||
report_file = LOGS_DIR / "probe_report.json"
|
||||
if not report_file.exists():
|
||||
return []
|
||||
|
||||
payload = json.loads(report_file.read_text(encoding="utf-8"))
|
||||
targets: list[str] = []
|
||||
for entity in payload.get("entities", []):
|
||||
if entity.get("status") == "ok" and entity.get("entity_set"):
|
||||
targets.append(str(entity["entity_set"]))
|
||||
return targets[:10]
|
||||
|
||||
|
||||
def _extract_link_map(records: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
field_map: dict[str, set[str]] = {}
|
||||
for row in records:
|
||||
for field in flatten_guid_like_fields(row):
|
||||
value = row.get(field)
|
||||
if value is None:
|
||||
continue
|
||||
field_map.setdefault(field, set()).add(str(value))
|
||||
|
||||
normalized: dict[str, list[str]] = {}
|
||||
for field, values in field_map.items():
|
||||
normalized[field] = sorted(values)[:5]
|
||||
return normalized
|
||||
|
||||
|
||||
def main() -> int:
|
||||
settings = load_settings()
|
||||
client = ODataClient(settings)
|
||||
targets = _load_probe_targets()
|
||||
|
||||
if not targets:
|
||||
print(
|
||||
"[error] probe_report.json not found or empty. "
|
||||
"Run `python -m odata_probe.probe_entities` first."
|
||||
)
|
||||
return 1
|
||||
|
||||
links_payload: dict[str, Any] = {
|
||||
"generated_at": utc_now_iso(),
|
||||
"service_root": settings.service_root,
|
||||
"entities": [],
|
||||
}
|
||||
|
||||
for entity_set in targets:
|
||||
try:
|
||||
records = client.read_entity_set_records(entity_set, top=min(settings.probe_top, 5))
|
||||
links_payload["entities"].append(
|
||||
{
|
||||
"entity_set": entity_set,
|
||||
"records_fetched": len(records),
|
||||
"link_field_samples": _extract_link_map(records),
|
||||
}
|
||||
)
|
||||
print(f"[ok] {entity_set}: link map generated")
|
||||
except Exception as exc:
|
||||
links_payload["entities"].append(
|
||||
{
|
||||
"entity_set": entity_set,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
print(f"[warn] {entity_set}: {exc}")
|
||||
|
||||
output_file = LOGS_DIR / "sample_links.json"
|
||||
output_file.write_text(
|
||||
json.dumps(links_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[ok] saved link map: {output_file}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from config.client import ODataClient, extract_entity_sets, utc_now_iso
|
||||
from config.settings import LOGS_DIR, load_settings
|
||||
|
||||
|
||||
def main() -> int:
|
||||
settings = load_settings()
|
||||
client = ODataClient(settings)
|
||||
|
||||
LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
metadata_xml = client.fetch_metadata()
|
||||
except Exception as exc:
|
||||
print(f"[error] failed to fetch metadata: {exc}")
|
||||
return 1
|
||||
|
||||
metadata_file = LOGS_DIR / "metadata.xml"
|
||||
metadata_file.write_text(metadata_xml, encoding="utf-8")
|
||||
|
||||
try:
|
||||
entity_sets = extract_entity_sets(metadata_xml)
|
||||
except Exception as exc:
|
||||
print(f"[error] metadata fetched, but parse failed: {exc}")
|
||||
return 1
|
||||
|
||||
summary = {
|
||||
"generated_at": utc_now_iso(),
|
||||
"service_root": settings.service_root,
|
||||
"metadata_url": settings.metadata_url,
|
||||
"entity_set_count": len(entity_sets),
|
||||
"entity_sets": entity_sets,
|
||||
}
|
||||
|
||||
entity_sets_file = LOGS_DIR / "entity_sets.json"
|
||||
entity_sets_file.write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"[ok] metadata saved: {metadata_file}")
|
||||
print(f"[ok] entity sets saved: {entity_sets_file}")
|
||||
print(f"[ok] total entity sets: {len(entity_sets)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from config.client import extract_entity_sets, utc_now_iso
|
||||
from config.settings import LOGS_DIR
|
||||
|
||||
|
||||
IMPORTANT_KEYWORDS = [
|
||||
"документ",
|
||||
"справочник",
|
||||
"регистр",
|
||||
"плансчетов",
|
||||
"хозрасчетный",
|
||||
"контрагенты",
|
||||
"договоры",
|
||||
"организации",
|
||||
"банковскиесчета",
|
||||
"document",
|
||||
"counterparty",
|
||||
"account",
|
||||
"posting",
|
||||
"contract",
|
||||
]
|
||||
|
||||
|
||||
def mark_priority(name: str) -> tuple[bool, list[str]]:
|
||||
lowered = name.lower().replace("_", "").replace(" ", "")
|
||||
matched = [kw for kw in IMPORTANT_KEYWORDS if kw in lowered]
|
||||
return bool(matched), matched
|
||||
|
||||
|
||||
def main() -> int:
|
||||
metadata_path = LOGS_DIR / "metadata.xml"
|
||||
if not metadata_path.exists():
|
||||
print(
|
||||
"[error] logs/metadata.xml not found. "
|
||||
"Run `python -m odata_probe.fetch_metadata` first."
|
||||
)
|
||||
return 1
|
||||
|
||||
metadata_xml = metadata_path.read_text(encoding="utf-8")
|
||||
entity_sets = extract_entity_sets(metadata_xml)
|
||||
|
||||
annotated: list[dict[str, object]] = []
|
||||
for item in entity_sets:
|
||||
priority, matched_keywords = mark_priority(item["name"])
|
||||
annotated.append(
|
||||
{
|
||||
**item,
|
||||
"priority": priority,
|
||||
"matched_keywords": matched_keywords,
|
||||
}
|
||||
)
|
||||
|
||||
output = {
|
||||
"generated_at": utc_now_iso(),
|
||||
"total": len(annotated),
|
||||
"priority_total": sum(1 for item in annotated if item["priority"]),
|
||||
"entity_sets": annotated,
|
||||
}
|
||||
|
||||
output_json = LOGS_DIR / "entity_sets_annotated.json"
|
||||
output_json.write_text(
|
||||
json.dumps(output, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
output_txt = LOGS_DIR / "entity_sets_annotated.txt"
|
||||
lines = []
|
||||
for item in annotated:
|
||||
flag = "*" if item["priority"] else " "
|
||||
matched = ", ".join(item["matched_keywords"]) if item["matched_keywords"] else "-"
|
||||
lines.append(f"{flag} {item['name']} | {item['entity_type']} | matched: {matched}")
|
||||
output_txt.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
print(f"[ok] total entity sets: {output['total']}")
|
||||
print(f"[ok] priority entity sets: {output['priority_total']}")
|
||||
print(f"[ok] saved: {output_json}")
|
||||
print(f"[ok] saved: {output_txt}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from config.client import ODataClient, flatten_guid_like_fields, utc_now_iso
|
||||
from config.settings import LOGS_DIR, load_settings
|
||||
|
||||
|
||||
TARGET_KEYWORDS = [
|
||||
"документ",
|
||||
"контрагент",
|
||||
"договор",
|
||||
"организац",
|
||||
"счет",
|
||||
"плансчетов",
|
||||
"регистр",
|
||||
"хозрасчет",
|
||||
"document",
|
||||
"counterparty",
|
||||
"contract",
|
||||
"account",
|
||||
"posting",
|
||||
]
|
||||
|
||||
|
||||
def _read_entity_sets() -> list[dict[str, Any]]:
|
||||
source_file = LOGS_DIR / "entity_sets_annotated.json"
|
||||
if source_file.exists():
|
||||
payload = json.loads(source_file.read_text(encoding="utf-8"))
|
||||
return payload.get("entity_sets", [])
|
||||
|
||||
fallback_file = LOGS_DIR / "entity_sets.json"
|
||||
if fallback_file.exists():
|
||||
payload = json.loads(fallback_file.read_text(encoding="utf-8"))
|
||||
return payload.get("entity_sets", [])
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _select_targets(entity_sets: list[dict[str, Any]], explicit_targets: tuple[str, ...]) -> list[str]:
|
||||
if explicit_targets:
|
||||
return list(dict.fromkeys(explicit_targets))
|
||||
|
||||
ranked: list[str] = []
|
||||
for item in entity_sets:
|
||||
name = str(item.get("name", ""))
|
||||
lowered = name.lower().replace("_", "")
|
||||
if any(keyword in lowered for keyword in TARGET_KEYWORDS):
|
||||
ranked.append(name)
|
||||
|
||||
if ranked:
|
||||
return list(dict.fromkeys(ranked[:20]))
|
||||
|
||||
return [str(item.get("name", "")) for item in entity_sets[:10] if item.get("name")]
|
||||
|
||||
|
||||
def _guess_link_fields(records: list[dict[str, Any]]) -> list[str]:
|
||||
fields: list[str] = []
|
||||
for record in records:
|
||||
fields.extend(flatten_guid_like_fields(record))
|
||||
return sorted(set(fields))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
settings = load_settings()
|
||||
entity_sets = _read_entity_sets()
|
||||
if not entity_sets:
|
||||
print(
|
||||
"[error] no entity sets found in logs. "
|
||||
"Run fetch_metadata and list_entity_sets first."
|
||||
)
|
||||
return 1
|
||||
|
||||
targets = _select_targets(entity_sets, settings.probe_entity_sets)
|
||||
if not targets:
|
||||
print("[error] no target entities to probe")
|
||||
return 1
|
||||
|
||||
client = ODataClient(settings)
|
||||
report_items: list[dict[str, Any]] = []
|
||||
success = 0
|
||||
|
||||
for entity_set in targets:
|
||||
try:
|
||||
records = client.read_entity_set_records(entity_set, top=settings.probe_top)
|
||||
field_names = sorted({field for row in records for field in row.keys()})
|
||||
link_fields = _guess_link_fields(records)
|
||||
sample_ids = []
|
||||
for row in records:
|
||||
for key in ("Ref_Key", "ID", "Id", "id"):
|
||||
value = row.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
sample_ids.append(value)
|
||||
break
|
||||
|
||||
report_items.append(
|
||||
{
|
||||
"entity_set": entity_set,
|
||||
"status": "ok",
|
||||
"records_fetched": len(records),
|
||||
"field_count": len(field_names),
|
||||
"fields": field_names,
|
||||
"suspected_link_fields": link_fields,
|
||||
"sample_ids": sample_ids[:10],
|
||||
}
|
||||
)
|
||||
success += 1
|
||||
print(f"[ok] {entity_set}: {len(records)} records, {len(link_fields)} link fields")
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
{
|
||||
"entity_set": entity_set,
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
print(f"[warn] {entity_set}: {exc}")
|
||||
|
||||
report = {
|
||||
"generated_at": utc_now_iso(),
|
||||
"service_root": settings.service_root,
|
||||
"probe_top": settings.probe_top,
|
||||
"targets_total": len(targets),
|
||||
"targets_success": success,
|
||||
"targets_failed": len(targets) - success,
|
||||
"entities": report_items,
|
||||
}
|
||||
|
||||
output_file = LOGS_DIR / "probe_report.json"
|
||||
output_file.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[ok] saved report: {output_file}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user