414 lines
15 KiB
Python
414 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""批量导入所有日出库数据（加固版）
|
||
|
||
改进：
|
||
1. 用 DictReader 代替索引取值，避免列偏移
|
||
2. 导入后自动校验货品编号空置率，超阈值自动回滚
|
||
3. 记录每条失败原因，不静默吞掉
|
||
4. 表头 strip 去空格/BOM，避免隐形字符匹配失败
|
||
"""
|
||
import sys, os, csv, json, subprocess, pymysql, importlib.util, re
|
||
from datetime import datetime
|
||
|
||
spec = importlib.util.spec_from_file_location("config", "config.py")
|
||
cfg = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(cfg)
|
||
DB_CONFIG = cfg.DB_CONFIG
|
||
TMP_DIR = "tmp_csv"
|
||
os.makedirs(TMP_DIR, exist_ok=True)
|
||
|
||
# 钉盘配置 - 限定到日出库文件夹
|
||
SPACE_ID = '28814565790'
|
||
OUTBOUND_FOLDER_ID = '3NwLYZXWynyBjRB4tZjwNxDvVkyEqBQm'
|
||
|
||
# ── 校验阈值 ──
|
||
EMPTY_CODE_THRESHOLD = 5 # 货品编号空置率超过此百分比(% )则回滚
|
||
|
||
def safe_float(val):
|
||
if not val: return 0.0
|
||
val = str(val).replace('"', '').replace('=', '').strip()
|
||
try:
|
||
return float(val)
|
||
except:
|
||
return 0.0
|
||
|
||
|
||
def safe_int(val):
|
||
if not val: return 0
|
||
val = str(val).replace('"', '').replace('=', '').strip()
|
||
try:
|
||
return int(float(val))
|
||
except:
|
||
return 0
|
||
|
||
|
||
def clean(val):
|
||
"""清洗单个字段值"""
|
||
if not val:
|
||
return ''
|
||
val = str(val).replace('"', '').replace('=', '').strip()
|
||
if val == '#N/A':
|
||
return ''
|
||
return val
|
||
|
||
|
||
def is_invalid_product_code(code):
|
||
"""过滤报表汇总行、占位编码和空编码。"""
|
||
normalized = clean(code).upper().rstrip(':')
|
||
return not normalized or normalized in ('NA', 'N/A', '合计')
|
||
|
||
def is_invalid_product_name(name):
|
||
"""过滤货品名称为单横线或双横线的占位数据。"""
|
||
return clean(name) in ('-', '--')
|
||
|
||
|
||
def clean_header(val):
|
||
"""清洗表头字段名 — 去 BOM、引号、等号、首尾空白"""
|
||
if not val:
|
||
return ''
|
||
s = str(val)
|
||
# 移除 UTF-8 BOM
|
||
s = s.replace('\ufeff', '')
|
||
s = s.replace('"', '').replace('=', '').strip()
|
||
return s
|
||
|
||
|
||
def parse_date_from_text(text):
|
||
"""从文件名提取日期"""
|
||
m = re.search(r'(\d{4})(\d{2})(\d{2})', text)
|
||
if m:
|
||
return f'{m.group(1)}-{m.group(2)}-{m.group(3)}'
|
||
return None
|
||
|
||
|
||
SOURCE_MAP = {
|
||
'wdt': '旺店通', 'jdbz': '京东泊紫', 'jdgj': '京东国际',
|
||
'jdjwt': '京东洁维塔', 'tmgj': '天猫国际',
|
||
# 钉盘文件实际使用 dymuto（用户口径 dymtuo），表示抖音M店。
|
||
'dymuto': '抖音M店', 'dymtuo': '抖音M店', 'jdsmj': '京东适美家',
|
||
'jdmrt': '京东美瑞图', 'dyqsimelr': '抖音Q店',
|
||
}
|
||
|
||
# 预期表头列名（区分全角/半角问题，只做参考）
|
||
WDT_EXPECTED_COLUMNS = ['货品编号', '货品名称', '品牌', '仓库', '店铺',
|
||
'一级类目', '二级类目', '三级类目', '货品数量', '货品成本']
|
||
|
||
|
||
def read_csv_rows(filepath):
|
||
"""尝试多种编码读取 CSV，返回 (header_list, data_rows_list) 或 (None, None)"""
|
||
for enc in ['utf-8-sig', 'gbk', 'gb18030']:
|
||
try:
|
||
with open(filepath, 'r', encoding=enc) as f:
|
||
reader = csv.reader(f)
|
||
raw_rows = list(reader)
|
||
if len(raw_rows) > 1:
|
||
return raw_rows[0], raw_rows[1:]
|
||
except:
|
||
continue
|
||
return None, None
|
||
|
||
|
||
def validate_import(cur, date_str, src):
|
||
"""导入后校验：货品编号空置率不能超过阈值"""
|
||
cur.execute("""
|
||
SELECT COUNT(*) AS total,
|
||
SUM(CASE WHEN 货品编号 IS NULL OR 货品编号 = '' THEN 1 ELSE 0 END) AS empty_code
|
||
FROM daily_outbound
|
||
WHERE 日期 = %s AND 数据来源 = %s
|
||
""", (date_str, src))
|
||
row = cur.fetchone()
|
||
total = row['total'] or 0
|
||
empty_code = row['empty_code'] or 0
|
||
if total > 0:
|
||
ratio = empty_code / total * 100
|
||
if ratio > EMPTY_CODE_THRESHOLD:
|
||
raise ValueError(
|
||
f"货品编号空置率 {ratio:.1f}% ({empty_code}/{total}) "
|
||
f"超过阈值 {EMPTY_CODE_THRESHOLD}%，已自动回滚"
|
||
)
|
||
return total, empty_code
|
||
|
||
|
||
# ── 第一步：获取文件列表（限定到日出库文件夹）──
|
||
print("📋 正在扫描日出库数据源文件夹...")
|
||
all_files = {} # name -> nodeId
|
||
cursor = None
|
||
|
||
while True:
|
||
args = ['dws', 'drive', 'list', '--space-id', SPACE_ID,
|
||
'--folder', OUTBOUND_FOLDER_ID, '--limit', '50', '--format', 'json']
|
||
if cursor:
|
||
args.extend(['--cursor', cursor])
|
||
|
||
result = subprocess.run(args, capture_output=True, text=True, timeout=20)
|
||
try:
|
||
data = json.loads(result.stdout).get('result', {})
|
||
except:
|
||
print(f" 解析失败: {result.stdout[:200]}")
|
||
break
|
||
|
||
for item in data.get('items', []):
|
||
if item.get('type') != 'FILE':
|
||
continue
|
||
name = item['name'].replace('.csv', '')
|
||
if not re.match(r'\d{8}', name):
|
||
continue
|
||
all_files[name] = item['fileId']
|
||
|
||
cursor = data.get('nextToken')
|
||
if not cursor:
|
||
break
|
||
|
||
print(f"找到 {len(all_files)} 个日出库文件")
|
||
|
||
# ── 第二步：按日期分组 ──
|
||
from collections import defaultdict
|
||
by_date = defaultdict(list)
|
||
for name in sorted(all_files.keys()):
|
||
parts = name.split('-')
|
||
date_str = parts[0]
|
||
src = parts[1]
|
||
by_date[date_str].append(src)
|
||
|
||
for d in sorted(by_date.keys()):
|
||
srcs = sorted(set(by_date[d]))
|
||
print(f" {d}: {srcs}")
|
||
|
||
# ── 第三步：查询已有记录 ──
|
||
conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT DISTINCT 日期, 数据来源 FROM daily_outbound")
|
||
existing = set()
|
||
for r in cur.fetchall():
|
||
d = r['日期'].strftime('%Y-%m-%d') if hasattr(
|
||
r['日期'], 'strftime') else str(r['日期'])[:10]
|
||
existing.add((d, r['数据来源']))
|
||
conn.close()
|
||
|
||
print(f"\n已有记录覆盖 {len(existing)} 个日期+来源组合")
|
||
|
||
# ── 第四步：逐个文件下载并导入 ──
|
||
total_imported = 0
|
||
total_files = 0
|
||
skipped = 0
|
||
errors = []
|
||
|
||
for name in sorted(all_files.keys()):
|
||
node_id = all_files[name]
|
||
parts = name.split('-')
|
||
date_str = parts[0]
|
||
src = parts[1]
|
||
src_label = SOURCE_MAP.get(src, src)
|
||
data_date = parse_date_from_text(name)
|
||
|
||
# 同时检测来源代码和中文名称，防止因名称不一致重复导入
|
||
src_label = SOURCE_MAP.get(src, src)
|
||
skip_reason = None
|
||
if data_date:
|
||
if (data_date, src) in existing:
|
||
skip_reason = f"来源代码({src})"
|
||
elif src != src_label and (data_date, src_label) in existing:
|
||
skip_reason = f"来源名称({src_label})"
|
||
if skip_reason:
|
||
print(f" ⏭️ 跳过已存在: {name} ({skip_reason})")
|
||
skipped += 1
|
||
continue
|
||
|
||
total_files += 1
|
||
print(f"\n[{total_files}] {name} ({src_label}, 日期:{data_date})")
|
||
|
||
# ─── 下载文件 ───
|
||
local_path = os.path.join(TMP_DIR, f"{name}.csv")
|
||
result = subprocess.run(
|
||
['dws', 'drive', 'download', '--node', node_id,
|
||
'--space-id', SPACE_ID, '--output', local_path, '--format', 'json'],
|
||
capture_output=True, text=True, timeout=60
|
||
)
|
||
if result.returncode != 0 or not os.path.exists(local_path) or os.path.getsize(local_path) < 10:
|
||
msg = f"下载失败: {result.stderr[:200]}"
|
||
print(f" ❌ {msg}")
|
||
errors.append((name, msg))
|
||
continue
|
||
|
||
# ─── 读取 CSV ───
|
||
header, data_rows = read_csv_rows(local_path)
|
||
if header is None or data_rows is None:
|
||
msg = f"读取CSV失败（编码/格式异常）"
|
||
print(f" ❌ {msg}")
|
||
errors.append((name, msg))
|
||
continue
|
||
|
||
print(f" 读取到 {len(data_rows)} 条记录, 列数: {len(header)}")
|
||
print(f" 表头: {header}")
|
||
|
||
# ── 清洗表头 ──
|
||
clean_headers = [clean_header(h) for h in header]
|
||
print(f" 清洗后: {clean_headers}")
|
||
|
||
# ─── 根据来源导入 ───
|
||
conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
|
||
inserted = 0
|
||
fail_count = 0
|
||
fail_samples = []
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
if src == 'wdt':
|
||
# ── wdt 完整格式 — 使用 DictReader 安全按列名取值 ──
|
||
# 重新以 DictReader 读取，确保列名与数据一一对应
|
||
for enc in ['utf-8-sig', 'gbk', 'gb18030']:
|
||
try:
|
||
f = open(local_path, 'r', encoding=enc)
|
||
dict_reader = csv.DictReader(f)
|
||
# 清洗 DictReader 的 fieldnames
|
||
dict_reader.fieldnames = [clean_header(h) for h in dict_reader.fieldnames]
|
||
break
|
||
except:
|
||
if 'f' in dir():
|
||
f.close()
|
||
continue
|
||
else:
|
||
raise RuntimeError("无法以任何编码读取CSV")
|
||
|
||
with f:
|
||
for row in dict_reader:
|
||
try:
|
||
product_code = clean(row.get('货品编号', ''))
|
||
# 关键校验：货品编号不能为空
|
||
if is_invalid_product_code(product_code):
|
||
fail_count += 1
|
||
if fail_count <= 3:
|
||
fail_samples.append(
|
||
f"行号? 无效货品编号={product_code}, 名称={row.get('货品名称','')[:20]}")
|
||
continue
|
||
|
||
product_name = clean(row.get('货品名称', ''))
|
||
if is_invalid_product_name(product_name):
|
||
fail_count += 1
|
||
continue
|
||
brand = clean(row.get('品牌', ''))
|
||
warehouse = clean(row.get('仓库', ''))
|
||
shop = clean(row.get('店铺', ''))
|
||
cat1 = clean(row.get('一级类目', ''))
|
||
cat2 = clean(row.get('二级类目', ''))
|
||
cat3 = clean(row.get('三级类目', ''))
|
||
qty = safe_int(row.get('货品数量', 0))
|
||
cost = safe_float(row.get('货品成本', 0))
|
||
|
||
cur.execute("""
|
||
INSERT INTO daily_outbound
|
||
(货品编号, 货品名称, 品牌, 仓库, 店铺, 一级类目, 二级类目, 三级类目, 货品数量, 货品成本, 日期, 数据来源, 数据类型)
|
||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
||
""", (product_code, product_name, brand, warehouse,
|
||
shop, cat1, cat2, cat3, qty, cost,
|
||
data_date, src, '出库数据'))
|
||
inserted += 1
|
||
except Exception as e:
|
||
fail_count += 1
|
||
if fail_count <= 3:
|
||
fail_samples.append(f"写入异常: {e}")
|
||
|
||
else:
|
||
# ── 简版格式（jdbz/jdgj/jdjwt/tmgj/dymuto/jdsmj）──
|
||
for raw_row in data_rows:
|
||
if not raw_row or len(raw_row) < 2:
|
||
fail_count += 1
|
||
continue
|
||
product_code = clean(raw_row[0])
|
||
qty = safe_int(raw_row[1])
|
||
if is_invalid_product_code(product_code):
|
||
fail_count += 1
|
||
continue
|
||
|
||
cur.execute("""
|
||
SELECT product_name, brand, category1, category2, category3, cost_price
|
||
FROM products WHERE product_code=%s LIMIT 1
|
||
""", (product_code,))
|
||
prod = cur.fetchone()
|
||
if prod:
|
||
pname = prod['product_name'] or ''
|
||
if is_invalid_product_name(pname):
|
||
fail_count += 1
|
||
continue
|
||
brand = prod['brand'] or ''
|
||
cat1 = prod['category1'] or ''
|
||
cat2 = prod['category2'] or ''
|
||
cat3 = prod['category3'] or ''
|
||
cost = float(prod['cost_price']) if prod['cost_price'] else 0
|
||
else:
|
||
pname = ''
|
||
brand = ''
|
||
cat1 = ''
|
||
cat2 = ''
|
||
cat3 = ''
|
||
cost = 0
|
||
|
||
try:
|
||
cur.execute("""
|
||
INSERT INTO daily_outbound
|
||
(货品编号, 货品名称, 品牌, 仓库, 店铺, 一级类目, 二级类目, 三级类目, 货品数量, 货品成本, 日期, 数据来源, 数据类型)
|
||
VALUES (%s,%s,%s,'','',%s,%s,%s,%s,%s,%s,%s,%s)
|
||
""", (product_code, pname, brand, cat1, cat2, cat3,
|
||
qty, cost, data_date, src, '出库数据'))
|
||
inserted += 1
|
||
except Exception as e:
|
||
fail_count += 1
|
||
if fail_count <= 3:
|
||
fail_samples.append(f"写入异常: {e}")
|
||
|
||
# ── 导入后校验 ──
|
||
validate_import(cur, data_date, src)
|
||
|
||
# 全部通过，提交
|
||
conn.commit()
|
||
|
||
except ValueError as ve:
|
||
# 校验失败 → 回滚
|
||
conn.rollback()
|
||
conn.close()
|
||
msg = f"❌ 校验失败: {ve}"
|
||
print(f" {msg}")
|
||
errors.append((name, str(ve)))
|
||
continue
|
||
|
||
except Exception as e:
|
||
conn.rollback()
|
||
conn.close()
|
||
msg = f"❌ 导入异常: {e}"
|
||
print(f" {msg}")
|
||
errors.append((name, str(e)))
|
||
continue
|
||
|
||
conn.close()
|
||
|
||
# ── 打印导入小结 ──
|
||
print(f" ✅ 导入 {inserted} 条", end='')
|
||
if fail_count > 0:
|
||
print(f", 跳过 {fail_count} 条异常记录")
|
||
for s in fail_samples[:3]:
|
||
print(f" 示例: {s}")
|
||
else:
|
||
print()
|
||
|
||
total_imported += inserted
|
||
|
||
# ── 最终汇总 ──
|
||
print(f"\n{'='*50}")
|
||
print(f"批量导入完成!")
|
||
print(f"新导入: {total_files} 个文件, {total_imported} 条记录")
|
||
print(f"已跳过: {skipped} 个已存在的文件")
|
||
if errors:
|
||
print(f"\n⚠️ 以下文件导入失败:")
|
||
for name, reason in errors:
|
||
print(f" - {name}: {reason}")
|
||
else:
|
||
# 导入脚本独立于 Gunicorn 进程运行，必须清除共享缓存文件，
|
||
# 让各 worker 放弃导入前的库存/计划预测结果。
|
||
try:
|
||
from services.cache import refresh_all_caches
|
||
refreshed = refresh_all_caches()
|
||
print(f"✅ 导入后计算缓存已清除（{len(refreshed)} 个模块）")
|
||
except Exception as e:
|
||
print(f"⚠️ 导入后计算缓存清除失败: {e}")
|
||
print(f"{'='*50}")
|