#!/usr/bin/env python3 """从钉盘CSV文件导入日出库数据 - 增量导入（默认）或全量替换（--force） 用法： python3 reimport.py # 增量：跳过已有的(日期+来源)，只导入新的 python3 reimport.py --force # 全量：清空表后重新导入全部（危险！） """ import csv, os, sys, pymysql, argparse from config import DB_CONFIG CSV_DIR = os.path.join(os.path.dirname(__file__), 'tmp_csv') # 来源简称 → 店铺名映射 SOURCE_SHOP = { 'wdt': '旺店通', 'jdbz': '京东泊紫', 'jdgj': '京东国际', 'jdjwt': '京东洁维塔', 'tmgj': '天猫国际', # 钉盘实际文件代码为 dymuto，兼容用户口径 dymtuo。 'dymuto': '抖音M店', 'dymtuo': '抖音M店', 'jdsmj': '京东适美家', 'jdmrt': '京东美瑞图', } SOURCE_TYPE = {'cksj': '出库数据'} SUPPORTED_SOURCES = set(SOURCE_SHOP) def clean_val(val, default=''): """清洗字段值：NA/NULL等占位符 → 默认值""" if val is None: return default s = str(val).strip() if s.upper() in ('NA', 'N/A', 'N.A', '#N/A', 'NULL', 'NONE', '-'): return default return s def is_invalid_product_code(code): """过滤报表汇总行、占位编码和空编码。""" normalized = clean_val(code).upper().rstrip(':') return not normalized or normalized in ('NA', 'N/A', '合计') def is_invalid_product_name(name): """过滤货品名称为单横线或双横线的占位数据。""" return str(name or '').strip() in ('-', '--') def parse_filename(name): """解析文件名: 20260518-jdjwt-cksj.csv → (date, source_key, type_key)""" base = name.replace('.csv', '') parts = base.split('-') raw_date = parts[0].strip() # 文件夹中存在历史误命名的 202607011-... 文件；仅接受标准 YYYYMMDD， # 避免把异常文件名作为非法日期写入数据库。 if len(raw_date) != 8 or not raw_date.isdigit(): raise ValueError(f'文件名日期格式异常: {name}') return raw_date, parts[1].strip(), parts[2].strip() def read_wdt(path, date_str, source_key): """读取旺店通完整格式（自动检测编码）""" # 先检测编码 detected_enc = None for enc in ['utf-8-sig', 'gbk', 'gb2312', 'utf-8']: try: with open(path, encoding=enc) as f: csv.DictReader(f).fieldnames detected_enc = enc break except (UnicodeDecodeError, UnicodeError): continue if not detected_enc: raise UnicodeDecodeError(f"无法解码文件: {path}") rows = [] with open(path, encoding=detected_enc) as f: reader = csv.DictReader(f) reader.fieldnames = [ h.replace('\ufeff', '').replace('"', '').replace('=', '').strip() for h in reader.fieldnames ] if reader.fieldnames else reader.fieldnames code_empty = 0 skip_summary = 0 for row in reader: product_code = clean_val(row.get('货品编号', '')) if is_invalid_product_code(product_code): code_empty += 1 continue if is_invalid_product_name(row.get('货品名称', '')): continue # 过滤合计行脏数据 if '合计' in product_code: skip_summary += 1 continue qty = int(row.get('货品数量', 0) or 0) if qty <= 0: continue brand = clean_val(row.get('品牌', '')) if '潘思宇代发' in brand: continue rows.append({ '日期': date_str, '货品编号': product_code, '货品名称': clean_val(row.get('货品名称', ''), '-'), '品牌': brand or '-', '仓库': clean_val(row.get('仓库', ''), '-'), '店铺': clean_val(row.get('店铺', ''), '-'), '一级类目': clean_val(row.get('一级类目', ''), '-'), '二级类目': clean_val(row.get('二级类目', ''), '-'), '三级类目': clean_val(row.get('三级类目', ''), '-'), '货品数量': qty, '货品成本': float(row.get('货品成本', 0) or 0), '数据来源': source_key, '数据类型': '出库数据', }) if skip_summary > 0: print(f" (已过滤 {skip_summary} 条合计行)") if code_empty > 0 and len(rows) > 0: ratio = code_empty / (len(rows) + code_empty) * 100 if ratio > 50: raise ValueError( f"货品编号空置率 {ratio:.1f}% ({code_empty}/{(len(rows) + code_empty)})" f"，编码可能检测错误（detected={detected_enc}），中止导入" ) return rows def read_simple(path, date_str, source_key, products_map): """读取简版格式（仅产品编码+数量），从products表补全""" rows = [] content = None for enc in ['utf-8-sig', 'gbk', 'gb2312', 'utf-8']: try: with open(path, encoding=enc) as f: content = f.read() break except UnicodeDecodeError: continue if content is None: print(f" ⚠️ 无法解码 {path}，跳过") return rows import io reader = csv.DictReader(io.StringIO(content)) reader.fieldnames = [ h.replace('\ufeff', '').replace('"', '').replace('=', '').strip() for h in reader.fieldnames ] if reader.fieldnames else reader.fieldnames for row in reader: code_keys = ['货品编号', '产品编码', '商品编码', '编码', 'product_code', 'goods_code'] qty_keys = ['货品数量', '数量', '出库量', 'qty', 'quantity'] code = '' for k in code_keys: v = clean_val(row.get(k, '')) if v: code = v break qty = 0 for k in qty_keys: v = clean_val(row.get(k, '')) if v: try: qty = int(float(v)) except: pass break if is_invalid_product_code(code) or qty <= 0: continue prod = products_map.get(code, {}) if is_invalid_product_name(prod.get('product_name', '')): continue rows.append({ '日期': date_str, '货品编号': code, '货品名称': prod.get('product_name', code), '品牌': prod.get('brand', '-'), '仓库': '-', '店铺': SOURCE_SHOP.get(source_key, source_key), '一级类目': prod.get('category1', '-'), '二级类目': prod.get('category2', '-'), '三级类目': prod.get('category3', '-'), '货品数量': qty, '货品成本': float(prod.get('cost_price', 0) or 0), '数据来源': source_key, '数据类型': '出库数据', }) return rows def main(): parser = argparse.ArgumentParser(description='导入日出库数据') parser.add_argument('--force', action='store_true', help='全量替换：清空 daily_outbound 表后重新导入所有数据') args = parser.parse_args() conn = pymysql.connect(**DB_CONFIG) cur = conn.cursor() # 1. 加载产品基础表 print("📦 加载产品基础表...") cur.execute("SELECT product_code, product_name, brand, category1, category2, category3, cost_price FROM products") products_map = {} for row in cur.fetchall(): products_map[row[0]] = { 'product_name': row[1] or '', 'brand': row[2] or '', 'category1': row[3] or '', 'category2': row[4] or '', 'category3': row[5] or '', 'cost_price': float(row[6]) if row[6] else 0, } print(f" 已加载 {len(products_map)} 个产品") # 2. 查询数据库中已有的(日期, 来源)组合 cur.execute("SELECT DISTINCT 日期, 数据来源 FROM daily_outbound") existing_set = set() for r in cur.fetchall(): d = r[0].strftime('%Y-%m-%d') if hasattr(r[0], 'strftime') else str(r[0])[:10] existing_set.add((d, r[1])) print(f" 数据库中已有 {len(existing_set)} 个 (日期+来源) 组合") # 3. 遍历CSV文件 all_rows = [] wdt_count = 0 simple_count = 0 skipped_existing = 0 new_files = 0 for fname in sorted(os.listdir(CSV_DIR)): if not fname.endswith('.csv') or 'cksj' not in fname: continue fpath = os.path.join(CSV_DIR, fname) try: date_str, source_key, type_key = parse_filename(fname) except ValueError as exc: print(f" ⚠️ 跳过文件: {fname} ({exc})") continue if source_key not in SUPPORTED_SOURCES: continue # ---- 如果是增量模式，跳过已有(日期+来源) ---- if not args.force: fmt_date = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:]}" if (fmt_date, source_key) in existing_set: print(f" ⏭️ 跳过已有: {fname}") skipped_existing += 1 continue new_files += 1 if source_key == 'wdt': rows = read_wdt(fpath, date_str, source_key) wdt_count += len(rows) print(f" 📄 {fname}: {len(rows)} 条 (旺店通)") else: rows = read_simple(fpath, date_str, source_key, products_map) simple_rows = rows rows = [r for r in rows if '潘思宇代发' not in (r.get('品牌', '') or '')] if len(rows) < len(simple_rows): print(f" (已过滤 {len(simple_rows) - len(rows)} 条潘思宇代发)") simple_count += len(rows) print(f" 📄 {fname}: {len(rows)} 条 ({SOURCE_SHOP.get(source_key, source_key)})") all_rows.extend(rows) # 如果是增量模式且没有新文件 if not args.force: print(f"\n📊 增量扫描结果: 新文件 {new_files} 个, 跳过已有 {skipped_existing} 个, 共 {len(all_rows)} 条") else: print(f"\n📊 全量模式: {len(all_rows)} 条 (旺店通 {wdt_count} + 其他 {simple_count})") if len(all_rows) == 0: print("✅ 没有新数据需要导入") conn.close() return # ---- 全量模式才清空 ---- if args.force: print("\n⚠️ --force 模式：清空旧数据...") cur.execute("TRUNCATE TABLE daily_outbound") conn.commit() print(" 已清空") # 4. 批量插入 print("💾 写入新数据...") batch_size = 500 cols = ['日期', '货品编号', '货品名称', '品牌', '仓库', '店铺', '一级类目', '二级类目', '三级类目', '货品数量', '货品成本', '数据来源', '数据类型'] placeholders = ','.join(['%s'] * len(cols)) sql = f"INSERT INTO daily_outbound ({','.join(cols)}) VALUES ({placeholders})" inserted = 0 for i in range(0, len(all_rows), batch_size): batch = all_rows[i:i + batch_size] batch_values = [] for r in batch: batch_values.append(tuple(r[c] for c in cols)) cur.executemany(sql, batch_values) conn.commit() inserted += len(batch) print(f" 已写入 {inserted}/{len(all_rows)} 条") # 5. 验证 cur.execute("SELECT COUNT(*) AS cnt, COALESCE(SUM(货品数量),0) AS total_qty, COALESCE(ROUND(SUM(货品成本),2),0) AS total_cost FROM daily_outbound") stats = cur.fetchone() print(f"\n✅ 导入完成!") print(f" 总记录数: {stats[0]}") print(f" 总出库量: {stats[1]}") print(f" 总成本: ¥{stats[2]}") cur.close() conn.close() if __name__ == '__main__': main()