198 lines
9.0 KiB
Python
198 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
||
"""2026-09-08 钉盘数据增量导入：日出库补录(09-04/05/06店铺+09-07旺店通) + 库存09-08"""
|
||
import csv, os, json, subprocess, pymysql, importlib.util, sys
|
||
from datetime import datetime
|
||
from collections import defaultdict
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
s = importlib.util.spec_from_file_location('config', 'config.py')
|
||
m = importlib.util.module_from_spec(s); s.loader.exec_module(m)
|
||
DB = m.DB_CONFIG
|
||
|
||
SPACE_ID = '28814565790'
|
||
OB_FOLDER = '3NwLYZXWynyBjRB4tZjwNxDvVkyEqBQm'
|
||
IV_FOLDER = 'NkDwLng8ZLGrXmrnFxYq3vRqVKMEvZBY'
|
||
TMP = 'tmp_csv/update_20260908'
|
||
os.makedirs(TMP, exist_ok=True)
|
||
|
||
def si(v):
|
||
try: return int(float(str(v or 0).replace('"','').replace('=','').strip()))
|
||
except: return 0
|
||
def sf(v):
|
||
try: return float(str(v or 0).replace('"','').replace('=','').strip())
|
||
except: return 0.0
|
||
def clean(v): return str(v or '').replace('"','').replace('=','').strip()
|
||
def bad(c):
|
||
n = clean(c).upper().rstrip(':')
|
||
return not n or n in ('NA','N/A','合计')
|
||
|
||
SPECIALS = {'POZ-FM-0184-TM-SM':.055,'POZ-XFS-9014-TM-MD':.062,'POZ-FM-9742-TM-MD':.181,
|
||
'POZ-MM-0580-TM-MD':.22,'POZ-SKS-3374-LD':.1,'POZ-SKS-3367-LD':.1,
|
||
'POZ-SKS-3350-LD':.1,'POZ-SKS-3343-LD':.1}
|
||
|
||
def dws_list(folder):
|
||
out=[]; cursor=None
|
||
while True:
|
||
args=['dws','drive','list','--space-id',SPACE_ID,'--folder',folder,'--limit','50','--format','json']
|
||
if cursor: args+=['--cursor',cursor]
|
||
r=subprocess.run(args,capture_output=True,text=True)
|
||
if r.returncode!=0:
|
||
print(' [list err]',r.stderr[:150]); break
|
||
d=json.loads(r.stdout).get('result',{})
|
||
out+=d.get('items',[])
|
||
cursor=d.get('nextToken')
|
||
if not cursor: break
|
||
return out
|
||
|
||
def dl(name, fileId):
|
||
dest=os.path.join(TMP,name)
|
||
if os.path.exists(dest) and os.path.getsize(dest)>10: return dest
|
||
r=subprocess.run(['dws','drive','download','--node',fileId,'--space-id',SPACE_ID,'--output',dest,'--format','json'],capture_output=True,text=True)
|
||
if r.returncode!=0 or not os.path.exists(dest) or os.path.getsize(dest)<10:
|
||
print(' [dl err]',name,r.stderr[:120]); return None
|
||
return dest
|
||
|
||
def read_rows_auto(path):
|
||
"""尝试多种编码，返回 (header_list, data_rows)"""
|
||
for enc in ['utf-8-sig','gbk','gb18030']:
|
||
try:
|
||
with open(path, encoding=enc) as f:
|
||
rr=list(csv.reader(f))
|
||
if rr: return rr[0], rr[1:]
|
||
except: continue
|
||
return None, None
|
||
|
||
def import_outbound(conn, date, src, path):
|
||
"""wdt 全格式 或 店铺2列简版；返回插入条数"""
|
||
with conn.cursor() as x:
|
||
x.execute('DELETE FROM daily_outbound WHERE 日期=%s AND 数据来源=%s',(date,src))
|
||
header, rows = read_rows_auto(path)
|
||
if rows is None:
|
||
print(' [read err]',path); return 0
|
||
inserted=0
|
||
with conn.cursor() as x:
|
||
if src == 'wdt':
|
||
# 全格式 DictReader（wdt 文件为 GBK）
|
||
with open(path, encoding='gb18030') as f:
|
||
for row in csv.DictReader(f):
|
||
code=clean(row.get('货品编号'))
|
||
if bad(code): continue
|
||
x.execute('INSERT INTO daily_outbound (货品编号,货品名称,品牌,仓库,店铺,一级类目,二级类目,三级类目,货品数量,货品成本,日期,数据来源,数据类型) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
|
||
(code,clean(row.get('货品名称')),clean(row.get('品牌')),clean(row.get('仓库')),clean(row.get('店铺')),
|
||
clean(row.get('一级类目')),clean(row.get('二级类目')),clean(row.get('三级类目')),
|
||
si(row.get('货品数量')),sf(row.get('货品成本')),date,src,'出库数据'))
|
||
inserted+=1
|
||
else:
|
||
# 店铺简版：跳过表头，col0=编码 col1=数量
|
||
for raw in rows:
|
||
if not raw or len(raw)<2: continue
|
||
code=clean(raw[0]); qty=si(raw[1])
|
||
if bad(code): continue
|
||
x.execute('SELECT product_name, brand, category1, category2, category3, cost_price FROM products WHERE product_code=%s LIMIT 1',(code,))
|
||
p=x.fetchone()
|
||
if p:
|
||
pname=p['product_name'] or ''
|
||
if clean(pname) in ('-','--'): continue
|
||
brand=p['brand'] or ''; c1=p['category1'] or ''; c2=p['category2'] or ''; c3=p['category3'] or ''
|
||
cost=float(p['cost_price']) if p['cost_price'] else 0
|
||
else:
|
||
pname=brand=c1=c2=c3=''; cost=0
|
||
x.execute('INSERT INTO daily_outbound (货品编号,货品名称,品牌,仓库,店铺,一级类目,二级类目,三级类目,货品数量,货品成本,日期,数据来源,数据类型) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
|
||
(code,pname,brand,'','',c1,c2,c3,qty,cost,date,src,'出库数据'))
|
||
inserted+=1
|
||
return inserted
|
||
|
||
def import_inventory(conn, date, path, wh):
|
||
with conn.cursor() as x:
|
||
x.execute('DELETE FROM inventory_detail WHERE data_date=%s AND warehouse_code=%s',(date,wh))
|
||
inserted=0
|
||
with conn.cursor() as x:
|
||
with open(path, encoding='utf-8-sig') as f:
|
||
for row in csv.DictReader(f):
|
||
code=clean(row.get('货品编号'))
|
||
if bad(code) or clean(row.get('货品名称')) in ('-','--'): continue
|
||
cost=SPECIALS.get(code, sf(row.get('成本价')))
|
||
ct=clean(row.get('单品创建时间')); dt=None
|
||
for fmt in ('%Y-%m-%d %H:%M:%S','%Y-%m-%d'):
|
||
try: dt=datetime.strptime(ct,fmt); break
|
||
except: pass
|
||
x.execute('INSERT INTO inventory_detail (data_date,source,warehouse_code,goods_code,product_code,product_name,brand,cost_price,available_qty,production_in_transit,purchase_in_transit,category1,category2,category3,created_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
|
||
(date,'wdt',wh,clean(row.get('商家编码')),code,clean(row.get('货品名称')),clean(row.get('品牌')),cost,
|
||
si(row.get('库存')),si(row.get('生产在途量')),si(row.get('采购在途')),
|
||
clean(row.get('一级类目')),clean(row.get('二级类目')),clean(row.get('三级类目')),dt))
|
||
inserted+=1
|
||
return inserted
|
||
|
||
def main():
|
||
print('扫描钉盘...')
|
||
ob = dws_list(OB_FOLDER)
|
||
iv = dws_list(IV_FOLDER)
|
||
# 组织文件
|
||
ob_files = {} # name -> fileId
|
||
for x in ob:
|
||
if x['type']=='FILE' and x['name'].endswith('.csv'):
|
||
ob_files[x['name']]=x['fileId']
|
||
iv_files = {}
|
||
for x in iv:
|
||
if x['type']=='FILE' and x['name'].endswith('.csv'):
|
||
iv_files[x['name']]=x['fileId']
|
||
|
||
# 现有出库组合
|
||
conn = pymysql.connect(**DB, cursorclass=pymysql.cursors.DictCursor)
|
||
existing_ob = set()
|
||
with conn.cursor() as x:
|
||
x.execute("SELECT 日期,数据来源 FROM daily_outbound")
|
||
for r in x.fetchall():
|
||
d = r['日期'].strftime('%Y-%m-%d') if hasattr(r['日期'],'strftime') else str(r['日期'])[:10]
|
||
existing_ob.add((d, r['数据来源']))
|
||
x.execute("SELECT DISTINCT data_date FROM inventory_detail")
|
||
existing_iv = set(str(r['data_date'])[:10] for r in x.fetchall())
|
||
|
||
# 目标：日出库 09-04 ~ 09-07 所有来源
|
||
need_ob = []
|
||
for name, fid in sorted(ob_files.items()):
|
||
p = name.split('-')
|
||
if len(p) < 3 or not p[0].isdigit(): continue
|
||
date = f'{p[0][:4]}-{p[0][4:6]}-{p[0][6:8]}'
|
||
if not (date >= '2026-09-04' and date <= '2026-09-07'): continue
|
||
src = p[1]
|
||
if (date, src) in existing_ob: continue
|
||
need_ob.append((name, fid, date, src))
|
||
|
||
# 目标：库存 09-08
|
||
need_iv = []
|
||
for name, fid in sorted(iv_files.items()):
|
||
p = name.split('-')
|
||
if not (p and p[0]=='20260908' and name.endswith('.csv')): continue
|
||
wh = name.rsplit('-',1)[1][:-4]
|
||
if '2026-09-08' in existing_iv and wh in [n.rsplit('-',1)[1][:-4] for n in iv_files if n.startswith('20260908') and '2026-09-08' in existing_iv]:
|
||
pass
|
||
need_iv.append((name, fid, '2026-09-08', wh))
|
||
|
||
print(f'需导入日出库 {len(need_ob)} 个文件, 库存 {len(need_iv)} 个文件')
|
||
|
||
# 下载并导入出库
|
||
ob_ins = 0
|
||
for name, fid, date, src in need_ob:
|
||
p = dl(name, fid)
|
||
if not p: continue
|
||
n = import_outbound(conn, date, src, p)
|
||
ob_ins += n
|
||
print(f' 出库 {name} ({date} {src}) 导入 {n} 条')
|
||
|
||
# 下载并导入库存 09-08
|
||
iv_ins = 0
|
||
for name, fid, date, wh in need_iv:
|
||
p = dl(name, fid)
|
||
if not p: continue
|
||
n = import_inventory(conn, date, p, wh)
|
||
iv_ins += n
|
||
print(f' 库存 {name} (仓库{wh}) 导入 {n} 条')
|
||
|
||
conn.commit()
|
||
conn.close()
|
||
print(f'\n完成: 出库新增 {ob_ins} 条, 库存新增 {iv_ins} 条')
|
||
|
||
if __name__ == '__main__':
|
||
main()
|