#!/usr/bin/env python3 """从钉盘增量导入库存数据 扫描钉盘「库存数据源文件」文件夹，找到数据库中尚未导入的日期，下载并导入。 用法： python3 import_inventory.py # 增量导入 python3 import_inventory.py --force # 全量替换（危险！先清空再导入） """ import os, sys, csv, json, subprocess, pymysql, re, argparse from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import importlib.util spec = importlib.util.spec_from_file_location("config", "config.py") config = importlib.util.module_from_spec(spec) spec.loader.exec_module(config) DB_CONFIG = config.DB_CONFIG TMP_DIR = "tmp_csv" os.makedirs(TMP_DIR, exist_ok=True) SPACE_ID = '28814565790' INVENTORY_FOLDER_ID = 'NkDwLng8ZLGrXmrnFxYq3vRqVKMEvZBY' def log(msg): print(msg) def parse_int(val): if val is None or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'): return 0 try: return int(float(val)) except (ValueError, TypeError): return 0 def parse_float(val): if val is None or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'): return 0.0 try: return float(val) except (ValueError, TypeError): return 0.0 def is_invalid_product_code(code): """过滤报表汇总行、占位编码和空编码。""" normalized = str(code or '').strip().upper().rstrip(':') return not normalized or normalized in ('NA', 'N/A', '合计') def is_invalid_product_name(name): """过滤货品名称为单横线或双横线的占位数据。""" return str(name or '').strip() in ('-', '--') def parse_datetime(val): if not val or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'): return None try: return datetime.strptime(val.strip(), '%Y-%m-%d %H:%M:%S') except ValueError: try: return datetime.strptime(val.strip(), '%Y-%m-%d') except ValueError: return None def extract_date_from_name(name): """从文件名提取日期: 20260612-wdt-kcmx-01 → 2026-06-12""" m = re.match(r'(\d{4})(\d{2})(\d{2})', name) if m: return f'{m.group(1)}-{m.group(2)}-{m.group(3)}' return None def extract_warehouse_code(name): """从文件名提取仓库代码: 20260612-wdt-kcmx-01 → 01""" m = re.search(r'kcmx-(\d+)$', name) return m.group(1) if m else '00' def list_inventory_files(): """列出钉盘库存文件夹下所有文件""" all_items = [] cursor = None while True: cmd = ['dws', 'drive', 'list', '--space-id', SPACE_ID, '--folder', INVENTORY_FOLDER_ID, '--limit', '50', '--format', 'json'] if cursor: cmd.extend(['--cursor', cursor]) result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) try: data = json.loads(result.stdout).get('result', {}) items = data.get('items', []) all_items.extend(items) cursor = data.get('nextToken') if not cursor: break except Exception as e: log(f" ❌ 列出文件失败: {e}") break return all_items def count_existing_dates(conn, date_str=None): """查询数据库中已有库存数据的日期""" with conn.cursor() as cur: if date_str: cur.execute("SELECT COUNT(*) as cnt FROM inventory_detail WHERE data_date=%s", (date_str,)) return cur.fetchone()['cnt'] else: cur.execute("SELECT DISTINCT data_date FROM inventory_detail") return sorted(set(r['data_date'].strftime('%Y-%m-%d') if hasattr(r['data_date'], 'strftime') else str(r['data_date'])[:10] for r in cur.fetchall())) def import_file(conn, local_path, data_date, warehouse_code, source='wdt'): """导入单个文件到数据库""" with conn.cursor() as cur: raw = open(local_path, 'rb').read() try: text = raw.decode('utf-8-sig') except UnicodeDecodeError: text = raw.decode('gb18030') rows = list(csv.DictReader(text.splitlines())) inserted = 0 for row in rows: product_code = (row.get('货品编号') or '').strip() if is_invalid_product_code(product_code): continue if is_invalid_product_name(row.get('货品名称')): continue try: cur.execute(""" INSERT INTO inventory_detail (data_date, source, warehouse_code, goods_code, product_code, product_name, brand, cost_price, available_qty, production_in_transit, purchase_in_transit, category1, category2, category3, created_time) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, ( data_date, source, warehouse_code, row.get('商家编码', '').strip(), product_code, row.get('货品名称', '').strip(), row.get('品牌', '').strip(), parse_float(row.get('成本价', 0)), parse_int(row.get('库存', row.get('可用库存', 0))), parse_int(row.get('生产在途量', 0)), parse_int(row.get('采购在途', 0)), row.get('一级类目', '').strip(), row.get('二级类目', '').strip(), row.get('三级类目', '').strip(), parse_datetime(row.get('单品创建时间', '')) )) inserted += 1 except Exception as e: if inserted < 3: log(f" 插入异常: {e}") return inserted def main(): parser = argparse.ArgumentParser(description='增量导入库存数据') parser.add_argument('--force', action='store_true', help='全量：先清空库存表再导入所有文件') args = parser.parse_args() conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor) # 1. 获取钉盘文件列表 log("📋 扫描库存数据源文件夹...") files = list_inventory_files() cvs_files = [f for f in files if f.get('extension') == 'csv'] log(f" 找到 {len(cvs_files)} 个 CSV 文件") # 按日期分组 from collections import defaultdict by_date = defaultdict(list) for f in cvs_files: name = f['name'].replace('.csv', '') d = extract_date_from_name(name) if d: by_date[d].append(f) sorted_dates = sorted(by_date.keys()) log(f" 共 {len(sorted_dates)} 个日期: {sorted_dates[-3:]}...(最近)") # 2. 查询已有日期 existing_dates = set(count_existing_dates(conn)) if args.force: log("\n⚠️ --force 模式：清空 inventory_detail 表...") with conn.cursor() as cur: cur.execute("DELETE FROM inventory_detail") conn.commit() log(" 已清空") target_dates = sorted_dates else: target_dates = [d for d in sorted_dates if d not in existing_dates] log(f"\n 数据库已有 {len(existing_dates)} 个日期，需导入 {len(target_dates)} 个新日期") if not target_dates: log("✅ 没有新数据需要导入") conn.close() return # 3. 逐个日期导入 total_imported = 0 total_files = 0 errors = [] for data_date in target_dates: date_files = by_date[data_date] log(f"\n📅 {data_date} ({len(date_files)} 个文件)") # 删除该日期的旧数据（防止重复导入） with conn.cursor() as cur: cur.execute("DELETE FROM inventory_detail WHERE data_date=%s", (data_date,)) deleted = cur.rowcount conn.commit() if deleted > 0: log(f" 🧹 已清除 {deleted} 条旧记录") for f in date_files: name = f['name'].replace('.csv', '') file_id = f['fileId'] wh_code = extract_warehouse_code(name) total_files += 1 # 下载 local_path = os.path.join(TMP_DIR, f"{name}.csv") result = subprocess.run( ["dws", "drive", "download", "--node", file_id, "--space-id", SPACE_ID, "--output", local_path, "--format", "json"], capture_output=True, text=True, timeout=60 ) if result.returncode != 0 or not os.path.exists(local_path) or os.path.getsize(local_path) < 50: log(f" ❌ 下载失败: {name}") errors.append((name, "下载失败")) continue size = os.path.getsize(local_path) print(f" 📄 仓库{wh_code} ({size//1024}KB)...", end=' ', flush=True) # 编码检查 file_result = subprocess.run(["file", local_path], capture_output=True, text=True, timeout=5) if 'ISO' in file_result.stdout or '8859' in file_result.stdout: utf8_path = local_path.replace('.csv', '_utf8.csv') subprocess.run(["iconv", "-f", "GBK", "-t", "UTF-8", local_path, "-o", utf8_path], capture_output=True, timeout=10) os.replace(utf8_path, local_path) count = import_file(conn, local_path, data_date, wh_code) conn.commit() log(f"✅ {count} 条") total_imported += count # 汇总 log(f"\n{'='*50}") log(f"库存导入完成!") log(f"处理 {len(target_dates)} 个日期, {total_files} 个文件, 导入 {total_imported} 条记录") if errors: log(f"\n⚠️ 失败:") for name, reason in errors[:5]: log(f" - {name}: {reason}") log(f"{'='*50}") conn.close() if __name__ == '__main__': main()