313 lines
12 KiB
Python
313 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""从钉盘CSV文件导入日出库数据 - 增量导入（默认）或全量替换（--force）
|
||
|
||
用法：
|
||
python3 reimport.py # 增量：跳过已有的(日期+来源)，只导入新的
|
||
python3 reimport.py --force # 全量：清空表后重新导入全部（危险！）
|
||
"""
|
||
import csv, os, sys, pymysql, argparse
|
||
from config import DB_CONFIG
|
||
|
||
CSV_DIR = os.path.join(os.path.dirname(__file__), 'tmp_csv')
|
||
|
||
# 来源简称 → 店铺名映射
|
||
SOURCE_SHOP = {
|
||
'wdt': '旺店通', 'jdbz': '京东泊紫', 'jdgj': '京东国际',
|
||
'jdjwt': '京东洁维塔', 'tmgj': '天猫国际',
|
||
# 钉盘实际文件代码为 dymuto，兼容用户口径 dymtuo。
|
||
'dymuto': '抖音M店', 'dymtuo': '抖音M店', 'jdsmj': '京东适美家',
|
||
'jdmrt': '京东美瑞图',
|
||
}
|
||
|
||
SOURCE_TYPE = {'cksj': '出库数据'}
|
||
SUPPORTED_SOURCES = set(SOURCE_SHOP)
|
||
|
||
def clean_val(val, default=''):
|
||
"""清洗字段值：NA/NULL等占位符 → 默认值"""
|
||
if val is None:
|
||
return default
|
||
s = str(val).strip()
|
||
if s.upper() in ('NA', 'N/A', 'N.A', '#N/A', 'NULL', 'NONE', '-'):
|
||
return default
|
||
return s
|
||
|
||
|
||
def is_invalid_product_code(code):
|
||
"""过滤报表汇总行、占位编码和空编码。"""
|
||
normalized = clean_val(code).upper().rstrip(':')
|
||
return not normalized or normalized in ('NA', 'N/A', '合计')
|
||
|
||
def is_invalid_product_name(name):
|
||
"""过滤货品名称为单横线或双横线的占位数据。"""
|
||
return str(name or '').strip() in ('-', '--')
|
||
|
||
|
||
def parse_filename(name):
|
||
"""解析文件名: 20260518-jdjwt-cksj.csv → (date, source_key, type_key)"""
|
||
base = name.replace('.csv', '')
|
||
parts = base.split('-')
|
||
raw_date = parts[0].strip()
|
||
# 文件夹中存在历史误命名的 202607011-... 文件；仅接受标准 YYYYMMDD，
|
||
# 避免把异常文件名作为非法日期写入数据库。
|
||
if len(raw_date) != 8 or not raw_date.isdigit():
|
||
raise ValueError(f'文件名日期格式异常: {name}')
|
||
return raw_date, parts[1].strip(), parts[2].strip()
|
||
|
||
def read_wdt(path, date_str, source_key):
|
||
"""读取旺店通完整格式（自动检测编码）"""
|
||
# 先检测编码
|
||
detected_enc = None
|
||
for enc in ['utf-8-sig', 'gbk', 'gb2312', 'utf-8']:
|
||
try:
|
||
with open(path, encoding=enc) as f:
|
||
csv.DictReader(f).fieldnames
|
||
detected_enc = enc
|
||
break
|
||
except (UnicodeDecodeError, UnicodeError):
|
||
continue
|
||
if not detected_enc:
|
||
raise UnicodeDecodeError(f"无法解码文件: {path}")
|
||
|
||
rows = []
|
||
with open(path, encoding=detected_enc) as f:
|
||
reader = csv.DictReader(f)
|
||
reader.fieldnames = [
|
||
h.replace('\ufeff', '').replace('"', '').replace('=', '').strip()
|
||
for h in reader.fieldnames
|
||
] if reader.fieldnames else reader.fieldnames
|
||
|
||
code_empty = 0
|
||
skip_summary = 0
|
||
for row in reader:
|
||
product_code = clean_val(row.get('货品编号', ''))
|
||
if is_invalid_product_code(product_code):
|
||
code_empty += 1
|
||
continue
|
||
if is_invalid_product_name(row.get('货品名称', '')):
|
||
continue
|
||
# 过滤合计行脏数据
|
||
if '合计' in product_code:
|
||
skip_summary += 1
|
||
continue
|
||
qty = int(row.get('货品数量', 0) or 0)
|
||
if qty <= 0:
|
||
continue
|
||
brand = clean_val(row.get('品牌', ''))
|
||
if '潘思宇代发' in brand:
|
||
continue
|
||
rows.append({
|
||
'日期': date_str,
|
||
'货品编号': product_code,
|
||
'货品名称': clean_val(row.get('货品名称', ''), '-'),
|
||
'品牌': brand or '-',
|
||
'仓库': clean_val(row.get('仓库', ''), '-'),
|
||
'店铺': clean_val(row.get('店铺', ''), '-'),
|
||
'一级类目': clean_val(row.get('一级类目', ''), '-'),
|
||
'二级类目': clean_val(row.get('二级类目', ''), '-'),
|
||
'三级类目': clean_val(row.get('三级类目', ''), '-'),
|
||
'货品数量': qty,
|
||
'货品成本': float(row.get('货品成本', 0) or 0),
|
||
'数据来源': source_key,
|
||
'数据类型': '出库数据',
|
||
})
|
||
|
||
if skip_summary > 0:
|
||
print(f" (已过滤 {skip_summary} 条合计行)")
|
||
if code_empty > 0 and len(rows) > 0:
|
||
ratio = code_empty / (len(rows) + code_empty) * 100
|
||
if ratio > 50:
|
||
raise ValueError(
|
||
f"货品编号空置率 {ratio:.1f}% ({code_empty}/{(len(rows) + code_empty)})"
|
||
f"，编码可能检测错误（detected={detected_enc}），中止导入"
|
||
)
|
||
return rows
|
||
|
||
def read_simple(path, date_str, source_key, products_map):
|
||
"""读取简版格式（仅产品编码+数量），从products表补全"""
|
||
rows = []
|
||
content = None
|
||
for enc in ['utf-8-sig', 'gbk', 'gb2312', 'utf-8']:
|
||
try:
|
||
with open(path, encoding=enc) as f:
|
||
content = f.read()
|
||
break
|
||
except UnicodeDecodeError:
|
||
continue
|
||
if content is None:
|
||
print(f" ⚠️ 无法解码 {path}，跳过")
|
||
return rows
|
||
|
||
import io
|
||
reader = csv.DictReader(io.StringIO(content))
|
||
reader.fieldnames = [
|
||
h.replace('\ufeff', '').replace('"', '').replace('=', '').strip()
|
||
for h in reader.fieldnames
|
||
] if reader.fieldnames else reader.fieldnames
|
||
|
||
for row in reader:
|
||
code_keys = ['货品编号', '产品编码', '商品编码', '编码', 'product_code', 'goods_code']
|
||
qty_keys = ['货品数量', '数量', '出库量', 'qty', 'quantity']
|
||
code = ''
|
||
for k in code_keys:
|
||
v = clean_val(row.get(k, ''))
|
||
if v:
|
||
code = v
|
||
break
|
||
qty = 0
|
||
for k in qty_keys:
|
||
v = clean_val(row.get(k, ''))
|
||
if v:
|
||
try:
|
||
qty = int(float(v))
|
||
except:
|
||
pass
|
||
break
|
||
if is_invalid_product_code(code) or qty <= 0:
|
||
continue
|
||
prod = products_map.get(code, {})
|
||
if is_invalid_product_name(prod.get('product_name', '')):
|
||
continue
|
||
rows.append({
|
||
'日期': date_str,
|
||
'货品编号': code,
|
||
'货品名称': prod.get('product_name', code),
|
||
'品牌': prod.get('brand', '-'),
|
||
'仓库': '-',
|
||
'店铺': SOURCE_SHOP.get(source_key, source_key),
|
||
'一级类目': prod.get('category1', '-'),
|
||
'二级类目': prod.get('category2', '-'),
|
||
'三级类目': prod.get('category3', '-'),
|
||
'货品数量': qty,
|
||
'货品成本': float(prod.get('cost_price', 0) or 0),
|
||
'数据来源': source_key,
|
||
'数据类型': '出库数据',
|
||
})
|
||
return rows
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='导入日出库数据')
|
||
parser.add_argument('--force', action='store_true', help='全量替换：清空 daily_outbound 表后重新导入所有数据')
|
||
args = parser.parse_args()
|
||
|
||
conn = pymysql.connect(**DB_CONFIG)
|
||
cur = conn.cursor()
|
||
|
||
# 1. 加载产品基础表
|
||
print("📦 加载产品基础表...")
|
||
cur.execute("SELECT product_code, product_name, brand, category1, category2, category3, cost_price FROM products")
|
||
products_map = {}
|
||
for row in cur.fetchall():
|
||
products_map[row[0]] = {
|
||
'product_name': row[1] or '',
|
||
'brand': row[2] or '',
|
||
'category1': row[3] or '',
|
||
'category2': row[4] or '',
|
||
'category3': row[5] or '',
|
||
'cost_price': float(row[6]) if row[6] else 0,
|
||
}
|
||
print(f" 已加载 {len(products_map)} 个产品")
|
||
|
||
# 2. 查询数据库中已有的(日期, 来源)组合
|
||
cur.execute("SELECT DISTINCT 日期, 数据来源 FROM daily_outbound")
|
||
existing_set = set()
|
||
for r in cur.fetchall():
|
||
d = r[0].strftime('%Y-%m-%d') if hasattr(r[0], 'strftime') else str(r[0])[:10]
|
||
existing_set.add((d, r[1]))
|
||
print(f" 数据库中已有 {len(existing_set)} 个 (日期+来源) 组合")
|
||
|
||
# 3. 遍历CSV文件
|
||
all_rows = []
|
||
wdt_count = 0
|
||
simple_count = 0
|
||
skipped_existing = 0
|
||
new_files = 0
|
||
|
||
for fname in sorted(os.listdir(CSV_DIR)):
|
||
if not fname.endswith('.csv') or 'cksj' not in fname:
|
||
continue
|
||
fpath = os.path.join(CSV_DIR, fname)
|
||
try:
|
||
date_str, source_key, type_key = parse_filename(fname)
|
||
except ValueError as exc:
|
||
print(f" ⚠️ 跳过文件: {fname} ({exc})")
|
||
continue
|
||
|
||
if source_key not in SUPPORTED_SOURCES:
|
||
continue
|
||
|
||
# ---- 如果是增量模式，跳过已有(日期+来源) ----
|
||
if not args.force:
|
||
fmt_date = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:]}"
|
||
if (fmt_date, source_key) in existing_set:
|
||
print(f" ⏭️ 跳过已有: {fname}")
|
||
skipped_existing += 1
|
||
continue
|
||
new_files += 1
|
||
|
||
if source_key == 'wdt':
|
||
rows = read_wdt(fpath, date_str, source_key)
|
||
wdt_count += len(rows)
|
||
print(f" 📄 {fname}: {len(rows)} 条 (旺店通)")
|
||
else:
|
||
rows = read_simple(fpath, date_str, source_key, products_map)
|
||
simple_rows = rows
|
||
rows = [r for r in rows if '潘思宇代发' not in (r.get('品牌', '') or '')]
|
||
if len(rows) < len(simple_rows):
|
||
print(f" (已过滤 {len(simple_rows) - len(rows)} 条潘思宇代发)")
|
||
simple_count += len(rows)
|
||
print(f" 📄 {fname}: {len(rows)} 条 ({SOURCE_SHOP.get(source_key, source_key)})")
|
||
|
||
all_rows.extend(rows)
|
||
|
||
# 如果是增量模式且没有新文件
|
||
if not args.force:
|
||
print(f"\n📊 增量扫描结果: 新文件 {new_files} 个, 跳过已有 {skipped_existing} 个, 共 {len(all_rows)} 条")
|
||
else:
|
||
print(f"\n📊 全量模式: {len(all_rows)} 条 (旺店通 {wdt_count} + 其他 {simple_count})")
|
||
|
||
if len(all_rows) == 0:
|
||
print("✅ 没有新数据需要导入")
|
||
conn.close()
|
||
return
|
||
|
||
# ---- 全量模式才清空 ----
|
||
if args.force:
|
||
print("\n⚠️ --force 模式：清空旧数据...")
|
||
cur.execute("TRUNCATE TABLE daily_outbound")
|
||
conn.commit()
|
||
print(" 已清空")
|
||
|
||
# 4. 批量插入
|
||
print("💾 写入新数据...")
|
||
batch_size = 500
|
||
cols = ['日期', '货品编号', '货品名称', '品牌', '仓库', '店铺',
|
||
'一级类目', '二级类目', '三级类目', '货品数量', '货品成本', '数据来源', '数据类型']
|
||
|
||
placeholders = ','.join(['%s'] * len(cols))
|
||
sql = f"INSERT INTO daily_outbound ({','.join(cols)}) VALUES ({placeholders})"
|
||
|
||
inserted = 0
|
||
for i in range(0, len(all_rows), batch_size):
|
||
batch = all_rows[i:i + batch_size]
|
||
batch_values = []
|
||
for r in batch:
|
||
batch_values.append(tuple(r[c] for c in cols))
|
||
cur.executemany(sql, batch_values)
|
||
conn.commit()
|
||
inserted += len(batch)
|
||
print(f" 已写入 {inserted}/{len(all_rows)} 条")
|
||
|
||
# 5. 验证
|
||
cur.execute("SELECT COUNT(*) AS cnt, COALESCE(SUM(货品数量),0) AS total_qty, COALESCE(ROUND(SUM(货品成本),2),0) AS total_cost FROM daily_outbound")
|
||
stats = cur.fetchone()
|
||
print(f"\n✅ 导入完成!")
|
||
print(f" 总记录数: {stats[0]}")
|
||
print(f" 总出库量: {stats[1]}")
|
||
print(f" 总成本: ¥{stats[2]}")
|
||
|
||
cur.close()
|
||
conn.close()
|
||
|
||
if __name__ == '__main__':
|
||
main()
|