dy-dashboard/import_20260908_cloud.py

#!/usr/bin/env python3
"""云端 2026-09-08 增量导入（离线版，从 /tmp/update_20260908 读CSV）"""
import csv, os, pymysql, importlib.util, sys
from datetime import datetime

sys.path.insert(0, '/opt/dy-dashboard')
s = importlib.util.spec_from_file_location('config', '/opt/dy-dashboard/config.py')
m = importlib.util.module_from_spec(s); s.loader.exec_module(m)
DB = m.DB_CONFIG
BASE = '/tmp/update_20260908'

def si(v):
    try: return int(float(str(v or 0).replace('"','').replace('=','').strip()))
    except: return 0
def sf(v):
    try: return float(str(v or 0).replace('"','').replace('=','').strip())
    except: return 0.0
def clean(v): return str(v or '').replace('"','').replace('=','').strip()
def bad(c):
    n = clean(c).upper().rstrip(':')
    return not n or n in ('NA','N/A','合计')

SPECIALS = {'POZ-FM-0184-TM-SM':.055,'POZ-XFS-9014-TM-MD':.062,'POZ-FM-9742-TM-MD':.181,
            'POZ-MM-0580-TM-MD':.22,'POZ-SKS-3374-LD':.1,'POZ-SKS-3367-LD':.1,
            'POZ-SKS-3350-LD':.1,'POZ-SKS-3343-LD':.1}

def import_outbound(conn, date, src, path):
    with conn.cursor() as x:
        x.execute('DELETE FROM daily_outbound WHERE 日期=%s AND 数据来源=%s',(date,src))
    inserted=0
    with conn.cursor() as x:
        if src == 'wdt':
            # wdt 文件编码可能为 UTF-8(BOM) 或 GBK，自动检测
            raw = open(path, 'rb').read()
            text = None
            for enc in ('utf-8-sig', 'gb18030', 'gbk'):
                try:
                    text = raw.decode(enc); break
                except UnicodeDecodeError:
                    continue
            if text is None:
                print('  [编码识别失败]', path); return 0
            import io as _io
            for row in csv.DictReader(_io.StringIO(text)):
                    code=clean(row.get('货品编号'))
                    if bad(code): continue
                    x.execute('INSERT INTO daily_outbound (货品编号,货品名称,品牌,仓库,店铺,一级类目,二级类目,三级类目,货品数量,货品成本,日期,数据来源,数据类型) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                        (code,clean(row.get('货品名称')),clean(row.get('品牌')),clean(row.get('仓库')),clean(row.get('店铺')),
                         clean(row.get('一级类目')),clean(row.get('二级类目')),clean(row.get('三级类目')),
                         si(row.get('货品数量')),sf(row.get('货品成本')),date,src,'出库数据'))
                    inserted+=1
        else:
            with open(path, encoding='utf-8-sig') as f:
                rr=list(csv.reader(f))
            for raw in rr[1:]:
                if not raw or len(raw)<2: continue
                code=clean(raw[0]); qty=si(raw[1])
                if bad(code): continue
                x.execute('SELECT product_name, brand, category1, category2, category3, cost_price FROM products WHERE product_code=%s LIMIT 1',(code,))
                p=x.fetchone()
                if p:
                    pname=p['product_name'] or ''
                    if clean(pname) in ('-','--'): continue
                    brand=p['brand'] or ''; c1=p['category1'] or ''; c2=p['category2'] or ''; c3=p['category3'] or ''
                    cost=float(p['cost_price']) if p['cost_price'] else 0
                else:
                    pname=brand=c1=c2=c3=''; cost=0
                x.execute('INSERT INTO daily_outbound (货品编号,货品名称,品牌,仓库,店铺,一级类目,二级类目,三级类目,货品数量,货品成本,日期,数据来源,数据类型) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                    (code,pname,brand,'','',c1,c2,c3,qty,cost,date,src,'出库数据'))
                inserted+=1
    return inserted

def import_inventory(conn, date, path, wh):
    with conn.cursor() as x:
        x.execute('DELETE FROM inventory_detail WHERE data_date=%s AND warehouse_code=%s',(date,wh))
    inserted=0
    # 库存文件可能是 UTF-8(BOM) 或 GB18030，按实际字节自动识别。
    raw = open(path, 'rb').read()
    text = None
    for enc in ('utf-8-sig', 'gb18030', 'gbk'):
        try:
            text = raw.decode(enc)
            break
        except UnicodeDecodeError:
            continue
    if text is None:
        print('  [编码识别失败]', path)
        return 0
    import io as _io
    with conn.cursor() as x:
        for row in csv.DictReader(_io.StringIO(text)):
                code=clean(row.get('货品编号'))
                if bad(code) or clean(row.get('货品名称')) in ('-','--'): continue
                cost=SPECIALS.get(code, sf(row.get('成本价')))
                ct=clean(row.get('单品创建时间')); dt=None
                for fmt in ('%Y-%m-%d %H:%M:%S','%Y-%m-%d'):
                    try: dt=datetime.strptime(ct,fmt); break
                    except: pass
                x.execute('INSERT INTO inventory_detail (data_date,source,warehouse_code,goods_code,product_code,product_name,brand,cost_price,available_qty,production_in_transit,purchase_in_transit,category1,category2,category3,created_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
                    (date,'wdt',wh,clean(row.get('商家编码')),code,clean(row.get('货品名称')),clean(row.get('品牌')),cost,
                     si(row.get('库存')),si(row.get('生产在途量')),si(row.get('采购在途')),
                     clean(row.get('一级类目')),clean(row.get('二级类目')),clean(row.get('三级类目')),dt))
                inserted+=1
    return inserted

def main():
    conn = pymysql.connect(**DB, cursorclass=pymysql.cursors.DictCursor)
    ob_ins=0; iv_ins=0
    for fn in sorted(os.listdir(BASE)):
        if not fn.endswith('.csv'): continue
        p = fn[:-4].split('-')
        if len(p)>=2 and p[0].isdigit() and len(p[0])==8:
            date = f'{p[0][:4]}-{p[0][4:6]}-{p[0][6:8]}'
            if p[1]=='wdt' and p[2]=='kcmx':
                wh = p[-1]; n = import_inventory(conn, date, os.path.join(BASE,fn), wh); iv_ins+=n
                print(f'  库存 {fn} 导入 {n}')
            else:
                src = p[1]; n = import_outbound(conn, date, src, os.path.join(BASE,fn)); ob_ins+=n
                print(f'  出库 {fn} ({date} {src}) 导入 {n}')
    conn.commit(); conn.close()
    print(f'完成: 出库 {ob_ins} 条, 库存 {iv_ins} 条  CLOUD DONE')

if __name__ == '__main__':
    main()