307 lines
11 KiB
Python
Executable File
307 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
蝶雅采购智能体平台 - 自动更新脚本
|
||
从钉钉云盘下载日出库数据及日库存数据，自动导入并刷新系统计算
|
||
|
||
用法：
|
||
python3 auto_update.py # 全量更新
|
||
python3 auto_update.py --download-only # 仅下载不导入
|
||
python3 auto_update.py --import-only # 仅导入不下载
|
||
python3 auto_update.py --clear-cache # 仅清除系统缓存
|
||
|
||
定时任务（每天 7:00/12:00/18:00 执行）：
|
||
0 7,12,18 * * * cd /home/gxd/.openclaw/workspace/dy-dashboard && python3 auto_update.py >> logs/auto_update.log 2>&1
|
||
"""
|
||
import os, sys, csv, json, glob, subprocess
|
||
from datetime import datetime, timedelta
|
||
from io import StringIO
|
||
|
||
# 项目根目录
|
||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
|
||
# 自动检测虚拟环境 Python
|
||
_PYTHON = os.path.join(BASE_DIR, 'venv', 'bin', 'python3')
|
||
if os.path.exists(_PYTHON):
|
||
PYTHON = _PYTHON
|
||
else:
|
||
PYTHON = 'python3'
|
||
CSV_DIR = os.path.join(BASE_DIR, 'tmp_csv')
|
||
LOG_DIR = os.path.join(BASE_DIR, 'logs')
|
||
DATA_DIR = os.path.join(CSV_DIR, 'downloads')
|
||
|
||
# 钉盘配置 - 请按实际路径修改
|
||
# 钉盘配置
|
||
SPACE_ID = '28814565790'
|
||
OUBOUND_FOLDER_ID = '3NwLYZXWynyBjRB4tZjwNxDvVkyEqBQm'
|
||
INVENTORY_FOLDER_ID = 'NkDwLng8ZLGrXmrnFxYq3vRqVKMEvZBY'
|
||
|
||
def log(msg):
|
||
"""带时间戳的日志输出"""
|
||
ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
print(f'[{ts}] {msg}')
|
||
|
||
def ensure_dirs():
|
||
"""确保目录存在"""
|
||
for d in [CSV_DIR, LOG_DIR, DATA_DIR]:
|
||
os.makedirs(d, exist_ok=True)
|
||
|
||
def download_from_dingtalk():
|
||
"""从钉钉云盘下载数据文件"""
|
||
log('=' * 60)
|
||
log('步骤1/4: 从钉钉云盘下载数据文件')
|
||
log('=' * 60)
|
||
|
||
downloaded = []
|
||
ensure_dirs()
|
||
|
||
def _download_files(folder_id, folder_name, target_dir):
|
||
"""列出并下载指定文件夹下的新 CSV 文件"""
|
||
log(f' 扫描{folder_name}...')
|
||
try:
|
||
result = subprocess.run(
|
||
['dws', 'drive', 'list', '--space-id', SPACE_ID,
|
||
'--folder', folder_id, '--limit', '50', '--format', 'json'],
|
||
capture_output=True, text=True, timeout=30
|
||
)
|
||
data = json.loads(result.stdout)
|
||
files = [f for f in data.get('items', []) if f.get('extension') == 'csv']
|
||
log(f' {folder_name}: {len(files)} 个 CSV 文件')
|
||
|
||
new_count = 0
|
||
for f in files:
|
||
fname = f.get('name', '')
|
||
file_id = f.get('fileId', '')
|
||
fpath = os.path.join(target_dir, fname)
|
||
|
||
# 如果文件已存在则跳过
|
||
if os.path.exists(fpath):
|
||
continue
|
||
|
||
# 下载新文件
|
||
log(f' 下载: {fname}')
|
||
subprocess.run(
|
||
['dws', 'drive', 'download', '--space-id', SPACE_ID,
|
||
'--file-id', file_id, '--output', target_dir],
|
||
capture_output=True, text=True, timeout=60
|
||
)
|
||
new_count += 1
|
||
|
||
if new_count > 0:
|
||
log(f' ✅ 下载 {new_count} 个新文件')
|
||
else:
|
||
log(f' ℹ️ 无新文件需下载')
|
||
return True
|
||
except Exception as e:
|
||
log(f' ⚠️ {folder_name}操作失败: {e}')
|
||
return False
|
||
|
||
# 下载日出库数据到 tmp_csv/
|
||
if _download_files(OUBOUND_FOLDER_ID, '日出库文件夹', CSV_DIR):
|
||
downloaded.append('outbound')
|
||
|
||
# 下载库存数据到 tmp_csv/
|
||
if _download_files(INVENTORY_FOLDER_ID, '库存文件夹', CSV_DIR):
|
||
downloaded.append('inventory')
|
||
|
||
if not downloaded:
|
||
log(f' ⚠️ 未找到任何数据文件')
|
||
|
||
return downloaded
|
||
|
||
def import_data():
|
||
"""导入数据到数据库（增量，不清表）"""
|
||
log('=' * 60)
|
||
log('步骤2/4: 导入数据到数据库（增量模式）')
|
||
log('=' * 60)
|
||
|
||
# 导入出库数据（reimport.py 默认增量，不清表）
|
||
log(' 📦 导入日出库数据...')
|
||
result = os.system(f'cd {BASE_DIR} && {PYTHON} reimport.py 2>&1')
|
||
if result == 0:
|
||
log(' ✅ 日出库数据导入完成')
|
||
else:
|
||
log(f' ⚠️ 日出库数据导入异常（code={result}）')
|
||
|
||
# 导入库存数据（使用 import_inventory.py 脚本）
|
||
log(' 📦 导入日库存数据...')
|
||
result = os.system(f'cd {BASE_DIR} && {PYTHON} import_inventory.py 2>&1')
|
||
if result == 0:
|
||
log(' ✅ 库存数据导入完成')
|
||
else:
|
||
log(f' ⚠️ 库存数据导入异常（code={result}）')
|
||
|
||
def refresh_calculations():
|
||
"""刷新所有计算数据（优先使用 RefreshableCache，再通过API触发）"""
|
||
log('=' * 60)
|
||
log('步骤3/4: 刷新系统计算数据')
|
||
log('=' * 60)
|
||
|
||
# 1. 使用 RefreshableCache 机制直接刷新内存缓存
|
||
try:
|
||
from services.cache import refresh_all_caches
|
||
results = refresh_all_caches()
|
||
log(f' ✅ 内存缓存刷新完成（{len(results)} 模块）')
|
||
except Exception as e:
|
||
log(f' ⚠️ 内存缓存刷新失败: {e}')
|
||
|
||
import urllib.request
|
||
import json as jsonlib
|
||
|
||
# 登录获取会话（用于验证API是否正常工作）
|
||
login_url = 'http://127.0.0.1:18888/api/login'
|
||
|
||
# API 预热使用独立环境变量，数据库中不再存放可还原的登录密码。
|
||
password = os.environ.get('DY_AUTO_UPDATE_PASSWORD', '')
|
||
if not password:
|
||
log(' ⚠️ 未配置 DY_AUTO_UPDATE_PASSWORD，跳过API触发')
|
||
return
|
||
|
||
import http.cookiejar
|
||
cj = http.cookiejar.CookieJar()
|
||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
|
||
|
||
try:
|
||
req = urllib.request.Request(
|
||
login_url,
|
||
data=jsonlib.dumps({'username': '甘旭东', 'password': password}).encode('utf-8'),
|
||
headers={'Content-Type': 'application/json'}
|
||
)
|
||
resp = opener.open(req, timeout=5)
|
||
result = jsonlib.loads(resp.read())
|
||
if result.get('code') != 0:
|
||
log(' ⚠️ 系统登录失败，跳过API触发')
|
||
return
|
||
log(' ✅ 系统登录成功')
|
||
except Exception:
|
||
log(' ⚠️ 系统登录失败，跳过API触发')
|
||
return
|
||
|
||
# 清除仪表盘概览缓存，下次访问自动重建
|
||
try:
|
||
url = 'http://127.0.0.1:18888/api/dashboard/refresh-cache'
|
||
resp = opener.open(url, timeout=5)
|
||
result = jsonlib.loads(resp.read())
|
||
if result.get('code') == 0:
|
||
log(f' ✅ 仪表盘概览缓存已清除')
|
||
except Exception as e:
|
||
log(f' ⚠️ 仪表盘概览缓存清除失败: {e}')
|
||
|
||
# 触发各页面API以预热缓存
|
||
api_calls = [
|
||
('/api/dashboard/stats', '仪表盘概览数据'),
|
||
('/api/planning/list?page=1&size=1', '计划预测数据'),
|
||
('/api/products/list?page=1&size=1', '产品基础表数据'),
|
||
]
|
||
|
||
for path, name in api_calls:
|
||
try:
|
||
url = f'http://127.0.0.1:18888{path}'
|
||
resp = opener.open(url, timeout=10)
|
||
result = jsonlib.loads(resp.read())
|
||
if result.get('code') == 0:
|
||
log(f' ✅ {name}刷新成功')
|
||
else:
|
||
log(f' ⚠️ {name}刷新异常')
|
||
except Exception as e:
|
||
log(f' ⚠️ {name}刷新失败: {e}')
|
||
|
||
def verify_data():
|
||
"""验证数据完整性"""
|
||
log('=' * 60)
|
||
log('步骤4/4: 验证数据完整性')
|
||
log('=' * 60)
|
||
|
||
import pymysql
|
||
from config import DB_CONFIG
|
||
|
||
try:
|
||
conn = pymysql.connect(**DB_CONFIG)
|
||
with conn.cursor() as cur:
|
||
# 检查出库数据
|
||
cur.execute("SELECT COUNT(*) AS cnt, COALESCE(SUM(货品数量),0) AS total FROM daily_outbound")
|
||
row = cur.fetchone()
|
||
log(f' 📊 日出库数据: {row[0]} 条记录, 共 {row[1]:.0f} 件')
|
||
|
||
# 检查库存数据
|
||
cur.execute("SELECT COUNT(DISTINCT product_code) AS cnt, COALESCE(SUM(available_qty),0) AS total FROM inventory_detail")
|
||
row = cur.fetchone()
|
||
log(f' 📊 日库存数据: {row[0]} 个SKU, 共 {row[1]:.0f} 件')
|
||
|
||
# 检查产品数据
|
||
cur.execute("SELECT COUNT(*) FROM products")
|
||
total = cur.fetchone()[0]
|
||
log(f' 📊 产品基础数据: {total} 条')
|
||
|
||
conn.close()
|
||
log(f' ✅ 数据完整性检查通过')
|
||
except Exception as e:
|
||
log(f' ❌ 数据完整性检查失败: {e}')
|
||
|
||
def clear_cache():
|
||
"""清除系统缓存（刷新 RefreshableCache）"""
|
||
log('=' * 60)
|
||
log('清除系统缓存')
|
||
log('=' * 60)
|
||
|
||
try:
|
||
from services.cache import refresh_all_caches
|
||
results = refresh_all_caches()
|
||
log(f' ✅ 已刷新 {len(results)} 个缓存模块')
|
||
except Exception as e:
|
||
log(f' ⚠️ 缓存刷新失败: {e}')
|
||
|
||
log(' ✅ 缓存刷新完成')
|
||
|
||
def main():
|
||
"""主流程"""
|
||
import argparse
|
||
|
||
parser = argparse.ArgumentParser(description='蝶雅采购智能体平台自动更新脚本')
|
||
parser.add_argument('--download-only', action='store_true', help='仅下载数据')
|
||
parser.add_argument('--import-only', action='store_true', help='仅导入数据')
|
||
parser.add_argument('--clear-cache', action='store_true', help='仅清除缓存')
|
||
args = parser.parse_args()
|
||
|
||
ensure_dirs()
|
||
|
||
start_time = datetime.now()
|
||
log(f'🚀 蝶雅采购智能体平台自动更新开始')
|
||
log(f' 时间: {start_time.strftime("%Y-%m-%d %H:%M:%S")}')
|
||
log('')
|
||
|
||
if args.clear_cache:
|
||
clear_cache()
|
||
log(f'\n✅ 缓存清除完成，耗时: {(datetime.now()-start_time).seconds}s')
|
||
return
|
||
|
||
if args.download_only:
|
||
download_from_dingtalk()
|
||
log(f'\n✅ 下载完成，耗时: {(datetime.now()-start_time).seconds}s')
|
||
return
|
||
|
||
if args.import_only:
|
||
import_data()
|
||
log(f'\n✅ 导入完成，耗时: {(datetime.now()-start_time).seconds}s')
|
||
return
|
||
|
||
# 全量更新
|
||
ensure_dirs()
|
||
|
||
downloaded = download_from_dingtalk()
|
||
if not downloaded:
|
||
log(f'⚠️ 未下载到新数据，尝试使用本地数据')
|
||
|
||
import_data()
|
||
refresh_calculations()
|
||
verify_data()
|
||
|
||
total_time = (datetime.now() - start_time).total_seconds()
|
||
log('')
|
||
log('=' * 60)
|
||
log(f'✅ 自动更新完成！总耗时: {total_time:.0f}s')
|
||
log(f' 完成时间: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}')
|
||
log('=' * 60)
|
||
|
||
if __name__ == '__main__':
|
||
main()
|