129 lines
5.6 KiB
Python
129 lines
5.6 KiB
Python
"""Build canonical supplier entities and anomaly links from raw anomaly data.
|
|
|
|
The script is intentionally additive: it never overwrites anomaly_cases.supplier.
|
|
Run after ``alembic upgrade head``. Existing master rows are reused by normalized
|
|
name; low-confidence mentions remain marked ``待确认`` for manual review.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
from collections import defaultdict
|
|
|
|
from sqlalchemy import delete, distinct, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.core.config import get_settings
|
|
from app.models.entities import AnomalyCase, AnomalyCaseSupplier, PurchaseOrderLine, SupplierMaster
|
|
from app.services.supplier_cleaning import infer_location, normalized_key, resolve_canonical_name, split_supplier_cell
|
|
|
|
|
|
async def clean(*, rebuild: bool = False) -> dict[str, int]:
|
|
settings = get_settings()
|
|
engine = create_async_engine(settings.database_url, pool_pre_ping=True)
|
|
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
stats = defaultdict(int)
|
|
try:
|
|
async with factory() as session:
|
|
if rebuild:
|
|
await session.execute(delete(AnomalyCaseSupplier))
|
|
await session.execute(delete(SupplierMaster))
|
|
stats["derived_tables_rebuilt"] = 1
|
|
masters = {row.normalized_name: row for row in (await session.execute(select(SupplierMaster))).scalars()}
|
|
order_names = [
|
|
value.strip()
|
|
for value in (await session.execute(select(distinct(PurchaseOrderLine.supplier)))).scalars()
|
|
if value and value.strip()
|
|
]
|
|
cases = (await session.execute(select(AnomalyCase))).scalars().all()
|
|
parsed: list[tuple[AnomalyCase, object]] = []
|
|
for case in cases:
|
|
mentions = split_supplier_cell(case.supplier, material_type=case.material_type)
|
|
for mention in mentions:
|
|
canonical_name, confirmed = resolve_canonical_name(mention.name, order_names)
|
|
key = normalized_key(canonical_name)
|
|
master = masters.get(key)
|
|
if master is None:
|
|
city, province = infer_location(canonical_name)
|
|
master = SupplierMaster(
|
|
canonical_name=canonical_name,
|
|
short_name=mention.name,
|
|
normalized_name=key,
|
|
city=city,
|
|
province=province,
|
|
supplier_type="供应商",
|
|
aliases=[mention.name],
|
|
source="采购订单匹配" if confirmed else "异常数据清洗",
|
|
review_status="已确认" if confirmed else "待确认",
|
|
)
|
|
session.add(master)
|
|
masters[key] = master
|
|
stats["masters_created"] += 1
|
|
if confirmed:
|
|
stats["masters_confirmed_from_orders"] += 1
|
|
elif mention.name not in master.aliases:
|
|
master.aliases = [*master.aliases, mention.name]
|
|
if len(mention.name) < len(master.short_name or master.canonical_name):
|
|
master.short_name = mention.name
|
|
parsed.append((case, mention))
|
|
stats["cases_scanned"] += 1
|
|
|
|
# Assign all master IDs in one database round trip.
|
|
await session.flush()
|
|
existing_links = {
|
|
(case_id, supplier_id, role)
|
|
for case_id, supplier_id, role in (
|
|
await session.execute(
|
|
select(
|
|
AnomalyCaseSupplier.case_id,
|
|
AnomalyCaseSupplier.supplier_id,
|
|
AnomalyCaseSupplier.supplier_role,
|
|
)
|
|
)
|
|
).all()
|
|
}
|
|
pending_links: set[tuple[int, int, str | None]] = set()
|
|
for case, mention in parsed:
|
|
canonical_name, _ = resolve_canonical_name(mention.name, order_names)
|
|
master = masters[normalized_key(canonical_name)]
|
|
link_key = (case.id, master.id, mention.role)
|
|
if link_key in existing_links or link_key in pending_links:
|
|
stats["links_skipped"] += 1
|
|
continue
|
|
session.add(
|
|
AnomalyCaseSupplier(
|
|
case_id=case.id,
|
|
supplier_id=master.id,
|
|
supplier_role=mention.role,
|
|
material_type=mention.material_type,
|
|
source_text=mention.source_text,
|
|
match_method=mention.method,
|
|
match_confidence=mention.confidence,
|
|
review_status="待确认",
|
|
)
|
|
)
|
|
pending_links.add(link_key)
|
|
stats["links_created"] += 1
|
|
await session.commit()
|
|
finally:
|
|
await engine.dispose()
|
|
return dict(stats)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--rebuild",
|
|
action="store_true",
|
|
help="clear and rebuild only the derived supplier master/link tables",
|
|
)
|
|
args = parser.parse_args()
|
|
result = asyncio.run(clean(rebuild=args.rebuild))
|
|
for key, value in result.items():
|
|
print(f"{key}={value}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|