From 5bdb2852e573353ebe3c631d26046face8cfb1d4 Mon Sep 17 00:00:00 2001 From: kx_guo Date: Thu, 10 Sep 2026 17:33:54 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E4=BE=9B=E5=BA=94=E5=95=86?= =?UTF-8?q?=E4=B8=BB=E6=95=B0=E6=8D=AE=E5=B9=B6=E4=BC=98=E5=8C=96=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=E5=90=91=E9=87=8F=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 ++ app/models/__init__.py | 4 + app/models/entities.py | 60 +++++++- app/schemas/suppliers.py | 8 ++ app/services/conversations.py | 36 ++++- app/services/database_risk_scoring.py | 19 ++- app/services/rag.py | 9 +- app/services/supplier_cleaning.py | 126 +++++++++++++++++ app/services/supplier_identity.py | 44 ++++++ app/services/suppliers.py | 83 ++++++++++-- ...20260910_0019_supplier_master_and_links.py | 62 +++++++++ scripts/backfill_embeddings.py | 52 ++++--- scripts/clean_suppliers.py | 128 ++++++++++++++++++ tests/test_models.py | 2 + tests/test_supplier_cleaning.py | 29 ++++ tests/test_suppliers.py | 4 + 16 files changed, 635 insertions(+), 40 deletions(-) create mode 100644 app/services/supplier_cleaning.py create mode 100644 app/services/supplier_identity.py create mode 100644 migrations/versions/20260910_0019_supplier_master_and_links.py create mode 100644 scripts/clean_suppliers.py create mode 100644 tests/test_supplier_cleaning.py diff --git a/README.md b/README.md index d0ccabb..d3631d0 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,15 @@ python scripts/import_excel_data.py ` 脚本只导入异常明细 Sheet 的 2,448 条记录,以及标准文件中的“包装/包材/法规/内料”四个分类 Sheet 共 219 条记录;“总表”不重复导入。脚本可重复执行,已导入行会按源文件、Sheet 和 Excel 行号自动跳过。原始异常文本和标准附件链接会完整保留,不做截断。 +供应商清洗采用“原始字段保留 + 主数据 + 多对多关联”方式,不覆盖 `anomaly_cases.supplier`。使用具备建表及对 `anomaly_cases` 建外键权限的数据库账号执行: + +```powershell +python -m alembic upgrade head +python scripts/clean_suppliers.py +``` + +脚本会拆分 `/`、`、`、逗号、分号及换行分隔的复合供应商,生成 `supplier_master` 和 `anomaly_case_suppliers`;自动结果默认处于“待确认”状态。管理员应补充主数据中的标准名称、别名、城市与省份后再标记为“已确认”。供应商搜索会优先匹配标准名称、简称、别名、城市和省份;迁移尚未执行时自动回退到原始字段搜索。 + 风险评估预览:`POST http://127.0.0.1:18474/api/v1/risk-predictions/preview`。指标值表示风险程度(0-100),接口返回的综合分按 PRD 转换为安全得分,分数越高风险越低;等级阈值为 80/60/40/20。 对话页真实 AI:配置 `RISK_LLM_API_KEY`、`RISK_LLM_BASE_URL` 和 `RISK_LLM_MODEL` 后,调用 `POST /api/v1/conversations/{id}/stream`。接口使用 OpenAI 兼容 Chat Completions 协议,先提取 `intent/category/specification/packaging/supplier/factory/stage/confidence` 语义 JSON,再通过 SSE 返回增量文本;最近 20 条消息作为上下文,完整助手消息、语义结果和风险评分会在流结束后持久化。也兼容参考项目的 `OPENAI_API_KEY`、`LLM_BASE_URL`、`LLM_MODEL` 配置名。未配置密钥时返回明确的 503,不生成模拟回复。 diff --git a/app/models/__init__.py b/app/models/__init__.py index 4eafc59..c07312e 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -2,6 +2,7 @@ from app.models.entities import ( AnomalyCase, + AnomalyCaseSupplier, Conversation, ConversationMessage, OperationAudit, @@ -13,11 +14,13 @@ from app.models.entities import ( RolePermission, StandardRule, SupplierProfile, + SupplierMaster, UserAccount, ) __all__ = [ "AnomalyCase", + "AnomalyCaseSupplier", "Conversation", "ConversationMessage", "OperationAudit", @@ -29,5 +32,6 @@ __all__ = [ "RolePermission", "StandardRule", "SupplierProfile", + "SupplierMaster", "UserAccount", ] diff --git a/app/models/entities.py b/app/models/entities.py index 253ca86..ea87d86 100644 --- a/app/models/entities.py +++ b/app/models/entities.py @@ -4,7 +4,7 @@ from datetime import date, datetime from decimal import Decimal from typing import Any -from sqlalchemy import BigInteger, Boolean, Date, DateTime, ForeignKey, Index, Integer, Numeric, String, Text, func +from sqlalchemy import BigInteger, Boolean, Date, DateTime, ForeignKey, Index, Integer, Numeric, String, Text, UniqueConstraint, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -59,6 +59,64 @@ class AnomalyCase(Base): standard_rule: Mapped["StandardRule | None"] = relationship(back_populates="anomaly_cases") review_records: Mapped[list["ReviewRecord"]] = relationship(back_populates="case") + supplier_links: Mapped[list["AnomalyCaseSupplier"]] = relationship( + back_populates="case", cascade="all, delete-orphan" + ) + + +class SupplierMaster(Base): + """Canonical supplier entity independent from raw workbook text.""" + + __tablename__ = "supplier_master" + __table_args__ = ( + Index("ix_supplier_master_city", "city"), + Index("ix_supplier_master_normalized_name", "normalized_name", unique=True), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + canonical_name: Mapped[str] = mapped_column(String(200), nullable=False) + short_name: Mapped[str | None] = mapped_column(String(100)) + normalized_name: Mapped[str] = mapped_column(String(200), nullable=False) + city: Mapped[str | None] = mapped_column(String(50)) + province: Mapped[str | None] = mapped_column(String(50)) + supplier_type: Mapped[str | None] = mapped_column(String(50)) + aliases: Mapped[list[str]] = mapped_column(JSONB, default=list, server_default="[]", nullable=False) + status: Mapped[str] = mapped_column(String(20), default="启用", server_default="启用", nullable=False) + source: Mapped[str | None] = mapped_column(String(100)) + review_status: Mapped[str] = mapped_column(String(20), default="待确认", server_default="待确认", nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now(), nullable=False + ) + + anomaly_links: Mapped[list["AnomalyCaseSupplier"]] = relationship( + back_populates="supplier", cascade="all, delete-orphan" + ) + + +class AnomalyCaseSupplier(Base): + """Many-to-many mapping from an anomaly case to canonical suppliers.""" + + __tablename__ = "anomaly_case_suppliers" + __table_args__ = ( + UniqueConstraint("case_id", "supplier_id", "supplier_role", name="uq_case_supplier_role"), + Index("ix_case_suppliers_supplier", "supplier_id"), + Index("ix_case_suppliers_case", "case_id"), + ) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + case_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("anomaly_cases.id", ondelete="CASCADE"), nullable=False) + supplier_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("supplier_master.id", ondelete="CASCADE"), nullable=False) + supplier_role: Mapped[str | None] = mapped_column(String(50)) + material_type: Mapped[str | None] = mapped_column(String(200)) + source_text: Mapped[str | None] = mapped_column(Text) + match_method: Mapped[str] = mapped_column(String(30), default="manual", server_default="manual", nullable=False) + match_confidence: Mapped[Decimal | None] = mapped_column(Numeric(5, 4)) + review_status: Mapped[str] = mapped_column(String(20), default="待确认", server_default="待确认", nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=False) + + case: Mapped[AnomalyCase] = relationship(back_populates="supplier_links") + supplier: Mapped[SupplierMaster] = relationship(back_populates="anomaly_links") class SupplierProfile(Base): diff --git a/app/schemas/suppliers.py b/app/schemas/suppliers.py index fb2ead1..eb640a3 100644 --- a/app/schemas/suppliers.py +++ b/app/schemas/suppliers.py @@ -9,6 +9,14 @@ class SupplierOption(ApiSchema): name: str anomaly_count: int tier: str + city: str | None = None + province: str | None = None + short_name: str | None = None + review_status: str | None = None + city: str | None = None + province: str | None = None + short_name: str | None = None + review_status: str | None = None class SupplierListData(ApiSchema): diff --git a/app/services/conversations.py b/app/services/conversations.py index 8408c3b..f8f523c 100644 --- a/app/services/conversations.py +++ b/app/services/conversations.py @@ -37,6 +37,7 @@ from app.schemas.risk_prediction import ( from app.services.database_risk_scoring import DatabaseRiskScoringService from app.services.llm_client import LLMClient from app.services.purchase_orders import calendar_year_window, supplier_order_counts +from app.services.supplier_identity import resolve_supplier from app.services.rag import AnomalyCaseRAG logger = logging.getLogger("dieya_risk_prediction.conversations") @@ -412,6 +413,9 @@ class ConversationService: ).all() ) supplier_name = self._infer_supplier_follow_up(text, supplier_names) + master_identity = await resolve_supplier(self.session, text) + if not supplier_name and master_identity: + supplier_name = master_identity.canonical_name supplier_name = supplier_name or self._infer_unknown_supplier_follow_up(text) if not supplier_name: return normalized @@ -421,7 +425,9 @@ class ConversationService: "risk_requested": True, "supplier": supplier_name, "confidence": max(float(normalized.get("confidence") or 0), 0.75), - "semantic_source": "database_supplier_match" + "semantic_source": "supplier_master_match" + if master_identity + else "database_supplier_match" if supplier_name in supplier_names else "standalone_new_supplier_fallback", } @@ -575,12 +581,19 @@ class ConversationService: match = re.search(r"(?:供应商|加工厂|工厂)\s*(?:是|用|为)?\s*([^,。!??]+)", text) name = match.group(1).strip() if match else "未指定" is_factory = any(word in text for word in ("加工厂", "工厂", "生产厂")) and "供应商" not in text + identity = ( + None + if is_factory or not hasattr(self, "session") + else await resolve_supplier(self.session, name) + ) + lookup_name = identity.short_name or identity.aliases[0] if identity else name + display_name = identity.canonical_name if identity else name column = AnomalyCase.factory if is_factory else AnomalyCase.supplier - anomaly_count, a_count, b_count = await self._supplier_history_stats(column, name) + anomaly_count, a_count, b_count = await self._supplier_history_stats(column, lookup_name) cooperation_count = ( await supplier_order_counts( self.session, - name, + display_name, self._supplier_history_start(), self._supplier_history_end(), ) @@ -592,7 +605,7 @@ class ConversationService: cooperation_display = cooperation_count if has_order_data else None problems = await self._top_anomaly_types( column, - name, + lookup_name, since=self._supplier_history_start(), ) party_type = "加工厂" if is_factory else "供应商" @@ -602,7 +615,7 @@ class ConversationService: database_score = await DatabaseRiskScoringService( self.session, as_of=datetime.now(UTC).date(), - ).score(supplier=name) + ).score(supplier=display_name) supplier_dimension = next( (item for item in database_score.dimensions if item.name == "供应商表现"), None, @@ -610,7 +623,7 @@ class ConversationService: return { "type": "supplier", "title": "供应商深度画像", - "name": name, + "name": display_name, "partyType": party_type, "anomalyCount": anomaly_count, "cooperationCount": cooperation_display, @@ -1039,6 +1052,15 @@ class ConversationService: factory = factory or supplier supplier = None supplier_or_factory = supplier or factory + supplier_identity = await resolve_supplier(self.session, supplier) + supplier_lookup = ( + supplier_identity.short_name or supplier_identity.aliases[0] + if supplier_identity + else supplier + ) + if supplier_identity: + supplier = supplier_identity.canonical_name + supplier_or_factory = supplier database_score = await DatabaseRiskScoringService( self.session, @@ -1059,7 +1081,7 @@ class ConversationService: category_count = await self._count_category_anomalies(category) supplier_count, supplier_a_count, supplier_b_count = await self._supplier_history_stats( AnomalyCase.supplier, - supplier, + supplier_lookup, ) factory_count = await self._count_anomalies(AnomalyCase.factory, factory) anomaly_count = supplier_count if supplier else factory_count diff --git a/app/services/database_risk_scoring.py b/app/services/database_risk_scoring.py index b495a6a..b627017 100644 --- a/app/services/database_risk_scoring.py +++ b/app/services/database_risk_scoring.py @@ -13,6 +13,7 @@ from app.domain.model_config import DEFAULT_FREQUENCY_CAP, DEFAULT_FREQUENCY_COE from app.domain.risk_engine import RiskLevel from app.models.entities import AnomalyCase, RiskModelConfig from app.services.purchase_orders import calendar_year_window, supplier_order_counts +from app.services.supplier_identity import resolve_supplier STAGES = ("需求发现", "产品立项", "产品开发", "量产上市", "产品生命管理") @@ -69,7 +70,9 @@ class DatabaseRiskScoringService: category_name = normalize_category(category) material_name = normalize_material(material) stage_name = normalize_stage(stage) - party_name = clean_value(supplier) + identity = await resolve_supplier(self.session, clean_value(supplier)) + party_name = identity.canonical_name if identity else clean_value(supplier) + party_names = identity.match_names if identity else ((party_name,) if party_name else ()) resolved_cooperation_count = ( cooperation_count @@ -77,7 +80,7 @@ class DatabaseRiskScoringService: else await self._supplier_order_count(party_name) ) dimensions = ( - self._supplier_dimension(source_rows, party_name, resolved_cooperation_count), + self._supplier_dimension(source_rows, party_name, resolved_cooperation_count, party_names), self._material_dimension(source_rows, material_name), self._category_dimension(source_rows, category_name), DimensionScore( @@ -95,7 +98,7 @@ class DatabaseRiskScoringService: reason="当前数据库没有检测结果与标准判定记录,按 PRD 中性值 50 分计入", ), self._stage_dimension(source_rows, stage_name), - self._defect_dimension(source_rows, category_name, material_name, party_name, defect_level), + self._defect_dimension(source_rows, category_name, material_name, party_name, defect_level, party_names), ) score = round(sum(item.score * item.weight / 100 for item in dimensions), 2) return DatabaseRiskScore(score=score, level=classify_score(score), dimensions=dimensions) @@ -148,6 +151,7 @@ class DatabaseRiskScoringService: rows: list[dict[str, Any]], party_name: str | None, cooperation_count: int | None = None, + party_names: tuple[str, ...] = (), ) -> DimensionScore: weight = self.weights["供应商表现"] if not party_name: @@ -158,7 +162,7 @@ class DatabaseRiskScoringService: status="default", reason="未提供供应商/加工厂,按 PRD 缺失维度默认 50 分", ) - matched = [row for row in rows if party_matches(row, party_name)] + matched = [row for row in rows if party_matches_any(row, party_names or (party_name,))] anomaly_count = len(matched) count = cooperation_count if cooperation_count is not None and cooperation_count > 0 else anomaly_count if count == 0: @@ -312,6 +316,7 @@ class DatabaseRiskScoringService: material: str | None, party_name: str | None, defect_level: str | None, + party_names: tuple[str, ...] = (), ) -> DimensionScore: weight = self.weights["缺陷等级"] if not category and not material and not party_name and not defect_level: @@ -331,7 +336,7 @@ class DatabaseRiskScoringService: or normalize_category(row.get("product_name")) == category ) and (not material or normalize_material(row.get("material_type")) == material) - and (not party_name or party_matches(row, party_name)) + and (not party_name or party_matches_any(row, party_names or (party_name,))) ] counts = { level: sum( @@ -433,6 +438,10 @@ def party_matches(row: dict[str, Any], party_name: str) -> bool: return party_name.lower() in supplier.lower() +def party_matches_any(row: dict[str, Any], party_names: tuple[str, ...]) -> bool: + return any(party_matches(row, name) for name in party_names) + + def classify_score(score: float) -> RiskLevel: if score >= 80: return RiskLevel.LOW diff --git a/app/services/rag.py b/app/services/rag.py index 29c70cc..ade10fc 100644 --- a/app/services/rag.py +++ b/app/services/rag.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import get_settings from app.models.entities import AnomalyCase from app.schemas.risk_prediction import SimilarCase +from app.services.supplier_identity import resolve_supplier logger = logging.getLogger("dieya_risk_prediction.rag") _embedding_semaphore = asyncio.Semaphore(1) @@ -53,6 +54,12 @@ class AnomalyCaseRAG: stage: str | None = None, anomaly_type: str | None = None, ) -> list[RetrievedCase]: + identity = await resolve_supplier(self.session, supplier) + supplier_match = ( + identity.short_name or identity.aliases[0] + if identity + else supplier + ) terms = self._terms(query, category, material, supplier, factory, stage, anomaly_type) rows = await self._load_rows( query, @@ -71,7 +78,7 @@ class AnomalyCaseRAG: query=query, category=category, material=material, - supplier=supplier, + supplier=supplier_match, factory=factory, stage=stage, anomaly_type=anomaly_type, diff --git a/app/services/supplier_cleaning.py b/app/services/supplier_cleaning.py new file mode 100644 index 0000000..041b6f1 --- /dev/null +++ b/app/services/supplier_cleaning.py @@ -0,0 +1,126 @@ +"""Pure helpers for splitting and normalizing raw supplier cells.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + + +_SEPARATOR_RE = re.compile(r"[//、,,;;|\\\n]+") +_LABEL_RE = re.compile(r"(?P