57 lines
3.1 KiB
Python
57 lines
3.1 KiB
Python
"""Manual date-scoped outbound refresh using the daily sync parser and transactional writer."""
|
|
import argparse
|
|
import fcntl
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
import ding_daily_sync as sync
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('day')
|
|
parser.add_argument('--packet')
|
|
parser.add_argument('--local', action='store_true')
|
|
args = parser.parse_args()
|
|
day = datetime.strptime(args.day, '%Y-%m-%d').date()
|
|
sync.STATE.mkdir(parents=True, exist_ok=True)
|
|
with open(sync.STATE / ('local.lock' if args.local else 'cloud.lock'), 'w') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
conn = sync.connect(args.local)
|
|
try:
|
|
if args.local:
|
|
packets = json.loads(gzip.decompress(Path(args.packet).read_bytes()))
|
|
else:
|
|
files = {}
|
|
for item in sync.list_files('outbound'):
|
|
match = sync.PATTERNS['outbound'].fullmatch(item.get('name', ''))
|
|
if not match or item.get('type') != 'FILE' or match[1] != day.strftime('%Y%m%d'):
|
|
continue
|
|
key = 'dymuto' if match[2] == 'dymtuo' else match[2]
|
|
if key in sync.SOURCES and (key not in files or item['modifyTime'] > files[key]['modifyTime']):
|
|
files[key] = item
|
|
assert files, 'No outbound files found; no data changed'
|
|
with conn.cursor() as cur:
|
|
cur.execute('SELECT product_code,product_name,brand,category1,category2,category3,cost_price FROM products')
|
|
products = {r['product_code']: r for r in cur.fetchall()}
|
|
packets = []
|
|
for key, item in sorted(files.items()):
|
|
dest = sync.STATE / 'downloads' / item['name']
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
sync.dws_command([sync.DWS,'drive','download','--node',item['fileId'],'--space-id',sync.SPACE,'--output',str(dest),'--format','json'])
|
|
assert dest.stat().st_size == item['fileSize'], 'Incomplete download'
|
|
packets.append({'kind':'outbound','date':str(day),'key':key,
|
|
'rows':sync.parse(dest,'outbound',str(day),key,products)})
|
|
sync.atomic(sync.STATE / ('manual-outbound-'+str(day)+'.json.gz'), gzip.compress(sync.encode(packets)))
|
|
print('missing sources (skipped):', sorted(sync.SOURCES-files.keys()), flush=True)
|
|
assert packets and all(p['kind']=='outbound' and p['date']==str(day) and p['key'] in sync.SOURCES for p in packets)
|
|
print('source rows:', {p['key']:len(p['rows']) for p in packets}, flush=True)
|
|
changed = sync.apply(conn, packets)
|
|
for p in packets:
|
|
assert sync.digest(sync.db_rows(conn,p),'outbound') == sync.digest(p['rows'],'outbound')
|
|
print(json.dumps({'date':str(day),'changed':changed,'sources':len(packets),
|
|
'total_rows':sum(len(p['rows']) for p in packets),
|
|
'verified':True,'cache_refreshed':True},ensure_ascii=False), flush=True)
|
|
finally:
|
|
conn.close()
|