79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
"""
|
||
数据自动清洗脚本 - 每次服务启动时自动执行
|
||
"""
|
||
import pymysql
|
||
|
||
def clean():
|
||
conn = pymysql.connect(host="127.0.0.1", port=3306, user="dy_app", password="dy_data_2026", database="dy_data", charset="utf8mb4")
|
||
cur = conn.cursor()
|
||
|
||
total = 0
|
||
|
||
# 清理规则1：删除脏数据行
|
||
delete_rules = [
|
||
("UPPER(TRIM(货品编号)) IN ('NA', 'N/A', '合计', '合计:') OR TRIM(货品编号) = ''", '出库汇总/占位行'),
|
||
("货品名称 = 'NA'", '货品名称为NA'),
|
||
("货品名称 = ''", '货品名称为空'),
|
||
("货品编号 = '合计:'", '合计行'),
|
||
("仓库 LIKE '%虚拟仓%'", '仓库含虚拟仓'),
|
||
("品牌 LIKE '%潘思宇代发%'", '潘思宇代发品牌(无效数据)'),
|
||
]
|
||
for condition, label in delete_rules:
|
||
cur.execute(f"DELETE FROM daily_outbound WHERE {condition}")
|
||
deleted = cur.rowcount
|
||
if deleted > 0:
|
||
print(f" 🗑️ {label}: 删除 {deleted} 条")
|
||
total += deleted
|
||
|
||
# 清理库存明细中的报表汇总行/占位编码
|
||
cur.execute("""
|
||
DELETE FROM inventory_detail
|
||
WHERE UPPER(TRIM(product_code)) IN ('NA', 'N/A', '合计', '合计:')
|
||
OR TRIM(product_code) = ''
|
||
""")
|
||
inventory_deleted = cur.rowcount
|
||
if inventory_deleted > 0:
|
||
print(f" 🗑️ 库存汇总/占位行: 删除 {inventory_deleted} 条")
|
||
total += inventory_deleted
|
||
|
||
# 清理规则2：修复负值/异常值（整数溢出导致的负数）
|
||
cur.execute("""
|
||
UPDATE daily_outbound
|
||
SET 货品数量 = 0
|
||
WHERE 货品数量 < -1000000 OR 货品数量 > 10000000
|
||
""")
|
||
overflow = cur.rowcount
|
||
if overflow > 0:
|
||
print(f" 🛠️ 异常数值修复: {overflow} 条(整数溢出)")
|
||
|
||
# 清理规则3：修复字段格式（去除 ="xxx" 的脏包裹）
|
||
cur.execute("""
|
||
UPDATE daily_outbound
|
||
SET 仓库 = TRIM(BOTH '"' FROM TRIM(LEADING '=' FROM 仓库))
|
||
WHERE 仓库 LIKE '=%'
|
||
""")
|
||
fixed = cur.rowcount
|
||
if fixed > 0:
|
||
print(f" 🛠️ 仓库格式修复: {fixed} 条(=号包裹)")
|
||
|
||
cur.execute("""
|
||
UPDATE daily_outbound
|
||
SET 店铺 = TRIM(BOTH '"' FROM TRIM(LEADING '=' FROM 店铺))
|
||
WHERE 店铺 LIKE '=%'
|
||
""")
|
||
fixed2 = cur.rowcount
|
||
if fixed2 > 0:
|
||
print(f" 🛠️ 店铺格式修复: {fixed2} 条(=号包裹)")
|
||
|
||
conn.commit()
|
||
if total > 0 or fixed > 0 or fixed2 > 0:
|
||
print(f" ✅ 本次清理 {total} 条脏数据，修复 {fixed + fixed2} 条格式")
|
||
cur.close()
|
||
conn.close()
|
||
return total
|
||
|
||
if __name__ == "__main__":
|
||
print("🔍 运行数据自动清洗...")
|
||
clean()
|
||
print(" 完成")
|