#!/usr/bin/env python3
"""批量导入所有日出库数据（加固版）

改进：
  1. 用 DictReader 代替索引取值，避免列偏移
  2. 导入后自动校验货品编号空置率，超阈值自动回滚
  3. 记录每条失败原因，不静默吞掉
  4. 表头 strip 去空格/BOM，避免隐形字符匹配失败
"""
import sys, os, csv, json, subprocess, pymysql, importlib.util, re
from datetime import datetime

spec = importlib.util.spec_from_file_location("config", "config.py")
cfg = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cfg)
DB_CONFIG = cfg.DB_CONFIG
TMP_DIR = "tmp_csv"
os.makedirs(TMP_DIR, exist_ok=True)

# 钉盘配置 - 限定到日出库文件夹
SPACE_ID = '28814565790'
OUTBOUND_FOLDER_ID = '3NwLYZXWynyBjRB4tZjwNxDvVkyEqBQm'

# ── 校验阈值 ──
EMPTY_CODE_THRESHOLD = 5  # 货品编号空置率超过此百分比(% )则回滚

def safe_float(val):
    if not val: return 0.0
    val = str(val).replace('"', '').replace('=', '').strip()
    try:
        return float(val)
    except:
        return 0.0


def safe_int(val):
    if not val: return 0
    val = str(val).replace('"', '').replace('=', '').strip()
    try:
        return int(float(val))
    except:
        return 0


def clean(val):
    """清洗单个字段值"""
    if not val:
        return ''
    val = str(val).replace('"', '').replace('=', '').strip()
    if val == '#N/A':
        return ''
    return val


def is_invalid_product_code(code):
    """过滤报表汇总行、占位编码和空编码。"""
    normalized = clean(code).upper().rstrip(':')
    return not normalized or normalized in ('NA', 'N/A', '合计')

def is_invalid_product_name(name):
    """过滤货品名称为单横线或双横线的占位数据。"""
    return clean(name) in ('-', '--')


def clean_header(val):
    """清洗表头字段名 — 去 BOM、引号、等号、首尾空白"""
    if not val:
        return ''
    s = str(val)
    # 移除 UTF-8 BOM
    s = s.replace('\ufeff', '')
    s = s.replace('"', '').replace('=', '').strip()
    return s


def parse_date_from_text(text):
    """从文件名提取日期"""
    m = re.search(r'(\d{4})(\d{2})(\d{2})', text)
    if m:
        return f'{m.group(1)}-{m.group(2)}-{m.group(3)}'
    return None


SOURCE_MAP = {
    'wdt': '旺店通', 'jdbz': '京东泊紫', 'jdgj': '京东国际',
    'jdjwt': '京东洁维塔', 'tmgj': '天猫国际',
    # 钉盘文件实际使用 dymuto（用户口径 dymtuo），表示抖音M店。
    'dymuto': '抖音M店', 'dymtuo': '抖音M店', 'jdsmj': '京东适美家',
    'jdmrt': '京东美瑞图', 'dyqsimelr': '抖音Q店',
}

# 预期表头列名（区分全角/半角问题，只做参考）
WDT_EXPECTED_COLUMNS = ['货品编号', '货品名称', '品牌', '仓库', '店铺',
                        '一级类目', '二级类目', '三级类目', '货品数量', '货品成本']


def read_csv_rows(filepath):
    """尝试多种编码读取 CSV，返回 (header_list, data_rows_list) 或 (None, None)"""
    for enc in ['utf-8-sig', 'gbk', 'gb18030']:
        try:
            with open(filepath, 'r', encoding=enc) as f:
                reader = csv.reader(f)
                raw_rows = list(reader)
            if len(raw_rows) > 1:
                return raw_rows[0], raw_rows[1:]
        except:
            continue
    return None, None


def validate_import(cur, date_str, src):
    """导入后校验：货品编号空置率不能超过阈值"""
    cur.execute("""
        SELECT COUNT(*) AS total,
               SUM(CASE WHEN 货品编号 IS NULL OR 货品编号 = '' THEN 1 ELSE 0 END) AS empty_code
        FROM daily_outbound
        WHERE 日期 = %s AND 数据来源 = %s
    """, (date_str, src))
    row = cur.fetchone()
    total = row['total'] or 0
    empty_code = row['empty_code'] or 0
    if total > 0:
        ratio = empty_code / total * 100
        if ratio > EMPTY_CODE_THRESHOLD:
            raise ValueError(
                f"货品编号空置率 {ratio:.1f}% ({empty_code}/{total}) "
                f"超过阈值 {EMPTY_CODE_THRESHOLD}%，已自动回滚"
            )
    return total, empty_code


# ── 第一步：获取文件列表（限定到日出库文件夹）──
print("📋 正在扫描日出库数据源文件夹...")
all_files = {}  # name -> nodeId
cursor = None

while True:
    args = ['dws', 'drive', 'list', '--space-id', SPACE_ID,
            '--folder', OUTBOUND_FOLDER_ID, '--limit', '50', '--format', 'json']
    if cursor:
        args.extend(['--cursor', cursor])

    result = subprocess.run(args, capture_output=True, text=True, timeout=20)
    try:
        data = json.loads(result.stdout).get('result', {})
    except:
        print(f"  解析失败: {result.stdout[:200]}")
        break

    for item in data.get('items', []):
        if item.get('type') != 'FILE':
            continue
        name = item['name'].replace('.csv', '')
        if not re.match(r'\d{8}', name):
            continue
        all_files[name] = item['fileId']

    cursor = data.get('nextToken')
    if not cursor:
        break

print(f"找到 {len(all_files)} 个日出库文件")

# ── 第二步：按日期分组 ──
from collections import defaultdict
by_date = defaultdict(list)
for name in sorted(all_files.keys()):
    parts = name.split('-')
    date_str = parts[0]
    src = parts[1]
    by_date[date_str].append(src)

for d in sorted(by_date.keys()):
    srcs = sorted(set(by_date[d]))
    print(f"  {d}: {srcs}")

# ── 第三步：查询已有记录 ──
conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
with conn.cursor() as cur:
    cur.execute("SELECT DISTINCT 日期, 数据来源 FROM daily_outbound")
    existing = set()
    for r in cur.fetchall():
        d = r['日期'].strftime('%Y-%m-%d') if hasattr(
            r['日期'], 'strftime') else str(r['日期'])[:10]
        existing.add((d, r['数据来源']))
conn.close()

print(f"\n已有记录覆盖 {len(existing)} 个日期+来源组合")

# ── 第四步：逐个文件下载并导入 ──
total_imported = 0
total_files = 0
skipped = 0
errors = []

for name in sorted(all_files.keys()):
    node_id = all_files[name]
    parts = name.split('-')
    date_str = parts[0]
    src = parts[1]
    src_label = SOURCE_MAP.get(src, src)
    data_date = parse_date_from_text(name)

    # 同时检测来源代码和中文名称，防止因名称不一致重复导入
    src_label = SOURCE_MAP.get(src, src)
    skip_reason = None
    if data_date:
        if (data_date, src) in existing:
            skip_reason = f"来源代码({src})"
        elif src != src_label and (data_date, src_label) in existing:
            skip_reason = f"来源名称({src_label})"
    if skip_reason:
        print(f"  ⏭️ 跳过已存在: {name} ({skip_reason})")
        skipped += 1
        continue

    total_files += 1
    print(f"\n[{total_files}] {name} ({src_label}, 日期:{data_date})")

    # ─── 下载文件 ───
    local_path = os.path.join(TMP_DIR, f"{name}.csv")
    result = subprocess.run(
        ['dws', 'drive', 'download', '--node', node_id,
         '--space-id', SPACE_ID, '--output', local_path, '--format', 'json'],
        capture_output=True, text=True, timeout=60
    )
    if result.returncode != 0 or not os.path.exists(local_path) or os.path.getsize(local_path) < 10:
        msg = f"下载失败: {result.stderr[:200]}"
        print(f"  ❌ {msg}")
        errors.append((name, msg))
        continue

    # ─── 读取 CSV ───
    header, data_rows = read_csv_rows(local_path)
    if header is None or data_rows is None:
        msg = f"读取CSV失败（编码/格式异常）"
        print(f"  ❌ {msg}")
        errors.append((name, msg))
        continue

    print(f"  读取到 {len(data_rows)} 条记录, 列数: {len(header)}")
    print(f"  表头: {header}")

    # ── 清洗表头 ──
    clean_headers = [clean_header(h) for h in header]
    print(f"  清洗后: {clean_headers}")

    # ─── 根据来源导入 ───
    conn = pymysql.connect(**DB_CONFIG, cursorclass=pymysql.cursors.DictCursor)
    inserted = 0
    fail_count = 0
    fail_samples = []

    try:
        with conn.cursor() as cur:
            if src == 'wdt':
                # ── wdt 完整格式 — 使用 DictReader 安全按列名取值 ──
                # 重新以 DictReader 读取，确保列名与数据一一对应
                for enc in ['utf-8-sig', 'gbk', 'gb18030']:
                    try:
                        f = open(local_path, 'r', encoding=enc)
                        dict_reader = csv.DictReader(f)
                        # 清洗 DictReader 的 fieldnames
                        dict_reader.fieldnames = [clean_header(h) for h in dict_reader.fieldnames]
                        break
                    except:
                        if 'f' in dir():
                            f.close()
                        continue
                else:
                    raise RuntimeError("无法以任何编码读取CSV")

                with f:
                    for row in dict_reader:
                        try:
                            product_code = clean(row.get('货品编号', ''))
                            # 关键校验：货品编号不能为空
                            if is_invalid_product_code(product_code):
                                fail_count += 1
                                if fail_count <= 3:
                                    fail_samples.append(
                                        f"行号? 无效货品编号={product_code}, 名称={row.get('货品名称','')[:20]}")
                                continue

                            product_name = clean(row.get('货品名称', ''))
                            if is_invalid_product_name(product_name):
                                fail_count += 1
                                continue
                            brand = clean(row.get('品牌', ''))
                            warehouse = clean(row.get('仓库', ''))
                            shop = clean(row.get('店铺', ''))
                            cat1 = clean(row.get('一级类目', ''))
                            cat2 = clean(row.get('二级类目', ''))
                            cat3 = clean(row.get('三级类目', ''))
                            qty = safe_int(row.get('货品数量', 0))
                            cost = safe_float(row.get('货品成本', 0))

                            cur.execute("""
                                INSERT INTO daily_outbound
                                (货品编号, 货品名称, 品牌, 仓库, 店铺, 一级类目, 二级类目, 三级类目, 货品数量, 货品成本, 日期, 数据来源, 数据类型)
                                VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                            """, (product_code, product_name, brand, warehouse,
                                  shop, cat1, cat2, cat3, qty, cost,
                                  data_date, src, '出库数据'))
                            inserted += 1
                        except Exception as e:
                            fail_count += 1
                            if fail_count <= 3:
                                fail_samples.append(f"写入异常: {e}")

            else:
                # ── 简版格式（jdbz/jdgj/jdjwt/tmgj/dymuto/jdsmj）──
                for raw_row in data_rows:
                    if not raw_row or len(raw_row) < 2:
                        fail_count += 1
                        continue
                    product_code = clean(raw_row[0])
                    qty = safe_int(raw_row[1])
                    if is_invalid_product_code(product_code):
                        fail_count += 1
                        continue

                    cur.execute("""
                        SELECT product_name, brand, category1, category2, category3, cost_price
                        FROM products WHERE product_code=%s LIMIT 1
                    """, (product_code,))
                    prod = cur.fetchone()
                    if prod:
                        pname = prod['product_name'] or ''
                        if is_invalid_product_name(pname):
                            fail_count += 1
                            continue
                        brand = prod['brand'] or ''
                        cat1 = prod['category1'] or ''
                        cat2 = prod['category2'] or ''
                        cat3 = prod['category3'] or ''
                        cost = float(prod['cost_price']) if prod['cost_price'] else 0
                    else:
                        pname = ''
                        brand = ''
                        cat1 = ''
                        cat2 = ''
                        cat3 = ''
                        cost = 0

                    try:
                        cur.execute("""
                            INSERT INTO daily_outbound
                            (货品编号, 货品名称, 品牌, 仓库, 店铺, 一级类目, 二级类目, 三级类目, 货品数量, 货品成本, 日期, 数据来源, 数据类型)
                            VALUES (%s,%s,%s,'','',%s,%s,%s,%s,%s,%s,%s,%s)
                        """, (product_code, pname, brand, cat1, cat2, cat3,
                              qty, cost, data_date, src, '出库数据'))
                        inserted += 1
                    except Exception as e:
                        fail_count += 1
                        if fail_count <= 3:
                            fail_samples.append(f"写入异常: {e}")

            # ── 导入后校验 ──
            validate_import(cur, data_date, src)

            # 全部通过，提交
            conn.commit()

    except ValueError as ve:
        # 校验失败 → 回滚
        conn.rollback()
        conn.close()
        msg = f"❌ 校验失败: {ve}"
        print(f"  {msg}")
        errors.append((name, str(ve)))
        continue

    except Exception as e:
        conn.rollback()
        conn.close()
        msg = f"❌ 导入异常: {e}"
        print(f"  {msg}")
        errors.append((name, str(e)))
        continue

    conn.close()

    # ── 打印导入小结 ──
    print(f"  ✅ 导入 {inserted} 条", end='')
    if fail_count > 0:
        print(f", 跳过 {fail_count} 条异常记录")
        for s in fail_samples[:3]:
            print(f"    示例: {s}")
    else:
        print()

    total_imported += inserted

# ── 最终汇总 ──
print(f"\n{'='*50}")
print(f"批量导入完成!")
print(f"新导入: {total_files} 个文件, {total_imported} 条记录")
print(f"已跳过: {skipped} 个已存在的文件")
if errors:
    print(f"\n⚠️ 以下文件导入失败:")
    for name, reason in errors:
        print(f"  - {name}: {reason}")
else:
    # 导入脚本独立于 Gunicorn 进程运行，必须清除共享缓存文件，
    # 让各 worker 放弃导入前的库存/计划预测结果。
    try:
        from services.cache import refresh_all_caches
        refreshed = refresh_all_caches()
        print(f"✅ 导入后计算缓存已清除（{len(refreshed)} 个模块）")
    except Exception as e:
        print(f"⚠️ 导入后计算缓存清除失败: {e}")
print(f"{'='*50}")