"""Build canonical supplier entities and anomaly links from raw anomaly data. The script is intentionally additive: it never overwrites anomaly_cases.supplier. Run after ``alembic upgrade head``. Existing master rows are reused by normalized name; low-confidence mentions remain marked ``待确认`` for manual review. """ from __future__ import annotations import argparse import asyncio from collections import defaultdict from sqlalchemy import delete, distinct, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.core.config import get_settings from app.models.entities import AnomalyCase, AnomalyCaseSupplier, PurchaseOrderLine, SupplierMaster from app.services.supplier_cleaning import infer_location, normalized_key, resolve_canonical_name, split_supplier_cell async def clean(*, rebuild: bool = False) -> dict[str, int]: settings = get_settings() engine = create_async_engine(settings.database_url, pool_pre_ping=True) factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) stats = defaultdict(int) try: async with factory() as session: if rebuild: await session.execute(delete(AnomalyCaseSupplier)) await session.execute(delete(SupplierMaster)) stats["derived_tables_rebuilt"] = 1 masters = {row.normalized_name: row for row in (await session.execute(select(SupplierMaster))).scalars()} order_names = [ value.strip() for value in (await session.execute(select(distinct(PurchaseOrderLine.supplier)))).scalars() if value and value.strip() ] cases = (await session.execute(select(AnomalyCase))).scalars().all() parsed: list[tuple[AnomalyCase, object]] = [] for case in cases: mentions = split_supplier_cell(case.supplier, material_type=case.material_type) for mention in mentions: canonical_name, confirmed = resolve_canonical_name(mention.name, order_names) key = normalized_key(canonical_name) master = masters.get(key) if master is None: city, province = infer_location(canonical_name) master = SupplierMaster( canonical_name=canonical_name, short_name=mention.name, normalized_name=key, city=city, province=province, supplier_type="供应商", aliases=[mention.name], source="采购订单匹配" if confirmed else "异常数据清洗", review_status="已确认" if confirmed else "待确认", ) session.add(master) masters[key] = master stats["masters_created"] += 1 if confirmed: stats["masters_confirmed_from_orders"] += 1 elif mention.name not in master.aliases: master.aliases = [*master.aliases, mention.name] if len(mention.name) < len(master.short_name or master.canonical_name): master.short_name = mention.name parsed.append((case, mention)) stats["cases_scanned"] += 1 # Assign all master IDs in one database round trip. await session.flush() existing_links = { (case_id, supplier_id, role) for case_id, supplier_id, role in ( await session.execute( select( AnomalyCaseSupplier.case_id, AnomalyCaseSupplier.supplier_id, AnomalyCaseSupplier.supplier_role, ) ) ).all() } pending_links: set[tuple[int, int, str | None]] = set() for case, mention in parsed: canonical_name, _ = resolve_canonical_name(mention.name, order_names) master = masters[normalized_key(canonical_name)] link_key = (case.id, master.id, mention.role) if link_key in existing_links or link_key in pending_links: stats["links_skipped"] += 1 continue session.add( AnomalyCaseSupplier( case_id=case.id, supplier_id=master.id, supplier_role=mention.role, material_type=mention.material_type, source_text=mention.source_text, match_method=mention.method, match_confidence=mention.confidence, review_status="待确认", ) ) pending_links.add(link_key) stats["links_created"] += 1 await session.commit() finally: await engine.dispose() return dict(stats) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--rebuild", action="store_true", help="clear and rebuild only the derived supplier master/link tables", ) args = parser.parse_args() result = asyncio.run(clean(rebuild=args.rebuild)) for key, value in result.items(): print(f"{key}={value}") if __name__ == "__main__": main()