261 lines
9.7 KiB
Python
261 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
|
"""从钉盘增量导入库存数据
|
|
|
|
扫描钉盘「库存数据源文件」文件夹，找到数据库中尚未导入的日期，下载并导入。
|
|
|
|
用法：
|
|
python3 import_inventory.py # 增量导入
|
|
python3 import_inventory.py --force # 全量替换（危险！先清空再导入）
|
|
"""
|
|
import os, sys, csv, json, subprocess, pymysql, re, argparse
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import importlib.util
|
|
spec = importlib.util.spec_from_file_location("config", "config.py")
|
|
config = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(config)
|
|
DB_CONFIG = config.DB_CONFIG
|
|
|
|
TMP_DIR = "tmp_csv"
|
|
os.makedirs(TMP_DIR, exist_ok=True)
|
|
|
|
SPACE_ID = '28814565790'
|
|
INVENTORY_FOLDER_ID = 'NkDwLng8ZLGrXmrnFxYq3vRqVKMEvZBY'
|
|
|
|
def log(msg):
|
|
print(msg)
|
|
|
|
def parse_int(val):
|
|
if val is None or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'):
|
|
return 0
|
|
try:
|
|
return int(float(val))
|
|
except (ValueError, TypeError):
|
|
return 0
|
|
|
|
def parse_float(val):
|
|
if val is None or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'):
|
|
return 0.0
|
|
try:
|
|
return float(val)
|
|
except (ValueError, TypeError):
|
|
return 0.0
|
|
|
|
def is_invalid_product_code(code):
|
|
"""过滤报表汇总行、占位编码和空编码。"""
|
|
normalized = str(code or '').strip().upper().rstrip(':')
|
|
return not normalized or normalized in ('NA', 'N/A', '合计')
|
|
|
|
def is_invalid_product_name(name):
|
|
"""过滤货品名称为单横线或双横线的占位数据。"""
|
|
return str(name or '').strip() in ('-', '--')
|
|
|
|
def parse_datetime(val):
|
|
if not val or val == '' or str(val).upper() in ('NA', '#N/A', 'N/A'):
|
|
return None
|
|
try:
|
|
return datetime.strptime(val.strip(), '%Y-%m-%d %H:%M:%S')
|
|
except ValueError:
|
|
try:
|
|
return datetime.strptime(val.strip(), '%Y-%m-%d')
|
|
except ValueError:
|
|
return None
|
|
|
|
def extract_date_from_name(name):
|
|
"""从文件名提取日期: 20260612-wdt-kcmx-01 → 2026-06-12"""
|
|
m = re.match(r'(\d{4})(\d{2})(\d{2})', name)
|
|
if m:
|
|
return f'{m.group(1)}-{m.group(2)}-{m.group(3)}'
|
|
return None
|
|
|
|
def extract_warehouse_code(name):
|
|
"""从文件名提取仓库代码: 20260612-wdt-kcmx-01 → 01"""
|
|
m = re.search(r'kcmx-(\d+)$', name)
|
|
return m.group(1) if m else '00'
|
|
|
|
def list_inventory_files():
|
|
"""列出钉盘库存文件夹下所有文件"""
|
|
all_items = []
|
|
cursor = None
|
|
while True:
|
|
cmd = ['dws', 'drive', 'list', '--space-id', SPACE_ID,
|
|
'--folder', INVENTORY_FOLDER_ID, '--limit', '50', '--format', 'json']
|
|
if cursor:
|
|
cmd.extend(['--cursor', cursor])
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
|
try:
|
|
data = json.loads(result.stdout).get('result', {})
|
|
items = data.get('items', [])
|
|
all_items.extend(items)
|
|
cursor = data.get('nextToken')
|
|
if not cursor:
|
|
break
|
|
except Exception as e:
|
|
log(f" ❌ 列出文件失败: {e}")
|
|
break
|
|
return all_items
|
|
|
|
def count_existing_dates(conn, date_str=None):
|
|
"""查询数据库中已有库存数据的日期"""
|
|
with conn.cursor() as cur:
|
|
if date_str:
|
|
cur.execute("SELECT COUNT(*) as cnt FROM inventory_detail WHERE data_date=%s", (date_str,))
|
|
return cur.fetchone()['cnt']
|
|
else:
|
|
cur.execute("SELECT DISTINCT data_date FROM inventory_detail")
|
|
return sorted(set(r['data_date'].strftime('%Y-%m-%d') if hasattr(r['data_date'], 'strftime') else str(r['data_date'])[:10] for r in cur.fetchall()))
|
|
|
|
def import_file(conn, local_path, data_date, warehouse_code, source='wdt'):
|
|
"""导入单个文件到数据库"""
|
|
with conn.cursor() as cur:
|
|
raw = open(local_path, 'rb').read()
|
|
try:
|
|
text = raw.decode('utf-8-sig')
|
|
except UnicodeDecodeError:
|
|
text = raw.decode('gb18030')
|
|
rows = list(csv.DictReader(text.splitlines()))
|
|
inserted = 0
|
|
for row in rows:
|
|
product_code = (row.get('货品编号') or '').strip()
|
|
if is_invalid_product_code(product_code):
|
|
continue
|
|
if is_invalid_product_name(row.get('货品名称')):
|
|
continue
|
|
try:
|
|
cur.execute("""
|
|
INSERT INTO inventory_detail
|
|
(data_date, source, warehouse_code, goods_code, product_code, product_name,
|
|
brand, cost_price, available_qty, production_in_transit, purchase_in_transit,
|
|
category1, category2, category3, created_time)
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
""", (
|
|
data_date, source, warehouse_code,
|
|
row.get('商家编码', '').strip(),
|
|
product_code,
|
|
row.get('货品名称', '').strip(),
|
|
row.get('品牌', '').strip(),
|
|
parse_float(row.get('成本价', 0)),
|
|
parse_int(row.get('库存', row.get('可用库存', 0))),
|
|
parse_int(row.get('生产在途量', 0)),
|
|
parse_int(row.get('采购在途', 0)),
|
|
row.get('一级类目', '').strip(),
|
|
row.get('二级类目', '').strip(),
|
|
row.get('三级类目', '').strip(),
|
|
parse_datetime(row.get('单品创建时间', ''))
|
|
))
|
|
inserted += 1
|
|
except Exception as e:
|
|
if inserted < 3:
|
|
log(f" 插入异常: {e}")
|
|
return inserted
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='增量导入库存数据')
|
|
parser.add_argument('--force', action='store_true', help='全量：先清空库存表再导入所有文件')
|
|
args = parser.parse_args()
|
|
|
|
conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
|
|
|
|
# 1. 获取钉盘文件列表
|
|
log("📋 扫描库存数据源文件夹...")
|
|
files = list_inventory_files()
|
|
cvs_files = [f for f in files if f.get('extension') == 'csv']
|
|
log(f" 找到 {len(cvs_files)} 个 CSV 文件")
|
|
|
|
# 按日期分组
|
|
from collections import defaultdict
|
|
by_date = defaultdict(list)
|
|
for f in cvs_files:
|
|
name = f['name'].replace('.csv', '')
|
|
d = extract_date_from_name(name)
|
|
if d:
|
|
by_date[d].append(f)
|
|
sorted_dates = sorted(by_date.keys())
|
|
log(f" 共 {len(sorted_dates)} 个日期: {sorted_dates[-3:]}...(最近)")
|
|
|
|
# 2. 查询已有日期
|
|
existing_dates = set(count_existing_dates(conn))
|
|
|
|
if args.force:
|
|
log("\n⚠️ --force 模式：清空 inventory_detail 表...")
|
|
with conn.cursor() as cur:
|
|
cur.execute("DELETE FROM inventory_detail")
|
|
conn.commit()
|
|
log(" 已清空")
|
|
target_dates = sorted_dates
|
|
else:
|
|
target_dates = [d for d in sorted_dates if d not in existing_dates]
|
|
log(f"\n 数据库已有 {len(existing_dates)} 个日期，需导入 {len(target_dates)} 个新日期")
|
|
|
|
if not target_dates:
|
|
log("✅ 没有新数据需要导入")
|
|
conn.close()
|
|
return
|
|
|
|
# 3. 逐个日期导入
|
|
total_imported = 0
|
|
total_files = 0
|
|
errors = []
|
|
|
|
for data_date in target_dates:
|
|
date_files = by_date[data_date]
|
|
log(f"\n📅 {data_date} ({len(date_files)} 个文件)")
|
|
|
|
# 删除该日期的旧数据（防止重复导入）
|
|
with conn.cursor() as cur:
|
|
cur.execute("DELETE FROM inventory_detail WHERE data_date=%s", (data_date,))
|
|
deleted = cur.rowcount
|
|
conn.commit()
|
|
if deleted > 0:
|
|
log(f" 🧹 已清除 {deleted} 条旧记录")
|
|
|
|
for f in date_files:
|
|
name = f['name'].replace('.csv', '')
|
|
file_id = f['fileId']
|
|
wh_code = extract_warehouse_code(name)
|
|
total_files += 1
|
|
|
|
# 下载
|
|
local_path = os.path.join(TMP_DIR, f"{name}.csv")
|
|
result = subprocess.run(
|
|
["dws", "drive", "download", "--node", file_id,
|
|
"--space-id", SPACE_ID, "--output", local_path, "--format", "json"],
|
|
capture_output=True, text=True, timeout=60
|
|
)
|
|
if result.returncode != 0 or not os.path.exists(local_path) or os.path.getsize(local_path) < 50:
|
|
log(f" ❌ 下载失败: {name}")
|
|
errors.append((name, "下载失败"))
|
|
continue
|
|
|
|
size = os.path.getsize(local_path)
|
|
print(f" 📄 仓库{wh_code} ({size//1024}KB)...", end=' ', flush=True)
|
|
|
|
# 编码检查
|
|
file_result = subprocess.run(["file", local_path], capture_output=True, text=True, timeout=5)
|
|
if 'ISO' in file_result.stdout or '8859' in file_result.stdout:
|
|
utf8_path = local_path.replace('.csv', '_utf8.csv')
|
|
subprocess.run(["iconv", "-f", "GBK", "-t", "UTF-8", local_path, "-o", utf8_path],
|
|
capture_output=True, timeout=10)
|
|
os.replace(utf8_path, local_path)
|
|
|
|
count = import_file(conn, local_path, data_date, wh_code)
|
|
conn.commit()
|
|
log(f"✅ {count} 条")
|
|
total_imported += count
|
|
|
|
# 汇总
|
|
log(f"\n{'='*50}")
|
|
log(f"库存导入完成!")
|
|
log(f"处理 {len(target_dates)} 个日期, {total_files} 个文件, 导入 {total_imported} 条记录")
|
|
if errors:
|
|
log(f"\n⚠️ 失败:")
|
|
for name, reason in errors[:5]:
|
|
log(f" - {name}: {reason}")
|
|
log(f"{'='*50}")
|
|
|
|
conn.close()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|