120 lines
4.3 KiB
Python
120 lines
4.3 KiB
Python
"""
|
||
钉盘数据同步脚本 - 下载最新CSV并导入数据库
|
||
"""
|
||
import os, sys, json, subprocess, urllib.request, time
|
||
|
||
BASE_DIR = "/home/gxd/.openclaw/workspace/dy-dashboard"
|
||
CSV_DIR = os.path.join(BASE_DIR, "tmp_csv")
|
||
os.makedirs(CSV_DIR, exist_ok=True)
|
||
|
||
def log(m): print(f"[{time.strftime('%H:%M:%S')}] {m}")
|
||
|
||
def dws(cmd):
|
||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||
return json.loads(r.stdout) if r.stdout else {}
|
||
|
||
def download_file(node_id, filename):
|
||
"""下载单个文件"""
|
||
log(f" 下载 {filename}...")
|
||
info = dws(f'dws doc download --node "{node_id}" --format json')
|
||
if not info.get("success"):
|
||
log(f" ❌ 获取下载链接失败: {filename}")
|
||
return False
|
||
url = info.get("resourceUrl", "")
|
||
if not url:
|
||
log(f" ❌ 无下载链接: {filename}")
|
||
return False
|
||
dest = os.path.join(CSV_DIR, filename)
|
||
try:
|
||
urllib.request.urlretrieve(url, dest)
|
||
log(f" ✅ {filename} ({os.path.getsize(dest)} bytes)")
|
||
return True
|
||
except Exception as e:
|
||
log(f" ❌ 下载失败: {filename} - {e}")
|
||
return False
|
||
|
||
# === 1. 下载日出库数据（May 27-30） ===
|
||
log("="*50)
|
||
log("下载日出库 CSV 文件")
|
||
|
||
outbound_files = [
|
||
# 日期, 文件名, nodeId
|
||
("20260530", "20260530-wdt-cksj.csv", "14dA3GK8gjMb34bZIKvEpKjEJ9ekBD76"),
|
||
("20260529", "20260529-wdt-cksj.csv", "9bN7RYPWdM970O7GtkRjBxeDVZd1wyK0"),
|
||
("20260528", "20260528-wdt-cksj.csv", "pYLaezmVNe3rBDrNCGNlnlD1WrMqPxX6"),
|
||
("20260528", "20260528-jdgj-cksj.csv", "MyQA2dXW7eKDkAD2C5d4XwbRJzlwrZgb"),
|
||
("20260528", "20260528-tmgj-cksj.csv", "4lgGw3P8vRrBDmBeUpnOy9RZ85daZ90D"),
|
||
("20260527", "20260527-wdt-cksj.csv", "Amq4vjg890vMpAMPCPYolXYxJ3kdP0wQ"),
|
||
("20260527", "20260527-jdjwt-cksj.csv", "ZQYprEoWonZ5lk52tDXkx1lQ81waOeDk"),
|
||
("20260527", "20260527-tmgj-cksj.csv", "3NwLYZXWynyBjRB4tZ0rwgodVkyEqBQm"),
|
||
("20260527", "20260527-jdgj-cksj.csv", "4lgGw3P8vRrBDmBeUpnEm5gK85daZ90D"),
|
||
("20260527", "20260527-jdbz-cksj.csv", "NDoBb60VLQbxqvxMtBnbgDngJlemrZQ3"),
|
||
]
|
||
|
||
for date, fname, nid in outbound_files:
|
||
download_file(nid, fname)
|
||
|
||
# === 2. 下载库存数据（May 31 - 各仓库）===
|
||
log("")
|
||
log("="*50)
|
||
log("下载库存 CSV 文件")
|
||
|
||
# 获取库存文件列表和nodeId
|
||
inventory_result = dws('dws doc list --folder "NkDwLng8ZLGrXmrnFxYq3vRqVKMEvZBY" --format json')
|
||
inventory_nodes = inventory_result.get("nodes", [])
|
||
|
||
# 只下载5月31日的文件
|
||
download_count = 0
|
||
for n in inventory_nodes:
|
||
name = n.get("name", "")
|
||
if name.startswith("20260531") and name.endswith(".csv"):
|
||
nid = n.get("nodeId")
|
||
fname = name if name.endswith(".csv") else name + ".csv"
|
||
if download_file(nid, fname):
|
||
download_count += 1
|
||
|
||
log(f"\n库存下载完成: {download_count} 个文件")
|
||
|
||
# === 3. 执行导入 ===
|
||
log("")
|
||
log("="*50)
|
||
log("开始导入数据到数据库")
|
||
log("="*50)
|
||
|
||
log("导入日出库数据...")
|
||
r = subprocess.run(f"cd {BASE_DIR} && python3 reimport.py", shell=True, capture_output=True, text=True, timeout=300)
|
||
log(r.stdout[-500:] if len(r.stdout) > 500 else r.stdout)
|
||
if r.returncode != 0:
|
||
log(f"⚠️ 出库导入异常: {r.stderr[-200:]}")
|
||
|
||
log("导入库存数据...")
|
||
r = subprocess.run(f"cd {BASE_DIR} && python3 batch_import_inventory.py", shell=True, capture_output=True, text=True, timeout=300)
|
||
log(r.stdout[-500:] if len(r.stdout) > 500 else r.stdout)
|
||
if r.returncode != 0:
|
||
log(f"⚠️ 库存导入异常: {r.stderr[-200:]}")
|
||
|
||
# === 4. 验证 ===
|
||
log("")
|
||
log("="*50)
|
||
log("验证数据")
|
||
|
||
import pymysql
|
||
from config import DB_CONFIG
|
||
conn = pymysql.connect(**DB_CONFIG)
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT MAX(日期) AS d, COUNT(*) AS cnt FROM daily_outbound")
|
||
r = cur.fetchone()
|
||
log(f"日出库: 最新日期={r[0]} 总记录={r[1]}")
|
||
|
||
cur.execute("SELECT MAX(data_date) AS d, COUNT(*) AS cnt FROM inventory_detail")
|
||
r = cur.fetchone()
|
||
log(f"库存: 最新日期={r[0]} 总记录={r[1]}")
|
||
conn.close()
|
||
|
||
# === 5. 重启服务刷新缓存 ===
|
||
log("")
|
||
log("重启服务刷新缓存...")
|
||
os.system("kill $(lsof -t -i:18888) 2>/dev/null; sleep 1")
|
||
os.system(f"cd {BASE_DIR} && nohup gunicorn -w 4 -k gevent -b 0.0.0.0:18888 --timeout 120 app:app > /dev/null 2>&1 &")
|
||
log("✅ 服务已重启，数据同步完成")
|