"""Manual date-scoped outbound refresh using the daily sync parser and transactional writer.""" import argparse import fcntl import gzip import hashlib import json from datetime import datetime from pathlib import Path import ding_daily_sync as sync parser = argparse.ArgumentParser() parser.add_argument('day') parser.add_argument('--packet') parser.add_argument('--local', action='store_true') args = parser.parse_args() day = datetime.strptime(args.day, '%Y-%m-%d').date() sync.STATE.mkdir(parents=True, exist_ok=True) with open(sync.STATE / ('local.lock' if args.local else 'cloud.lock'), 'w') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) conn = sync.connect(args.local) try: if args.local: packets = json.loads(gzip.decompress(Path(args.packet).read_bytes())) else: files = {} for item in sync.list_files('outbound'): match = sync.PATTERNS['outbound'].fullmatch(item.get('name', '')) if not match or item.get('type') != 'FILE' or match[1] != day.strftime('%Y%m%d'): continue key = 'dymuto' if match[2] == 'dymtuo' else match[2] if key in sync.SOURCES and (key not in files or item['modifyTime'] > files[key]['modifyTime']): files[key] = item assert files, 'No outbound files found; no data changed' with conn.cursor() as cur: cur.execute('SELECT product_code,product_name,brand,category1,category2,category3,cost_price FROM products') products = {r['product_code']: r for r in cur.fetchall()} packets = [] for key, item in sorted(files.items()): dest = sync.STATE / 'downloads' / item['name'] dest.parent.mkdir(parents=True, exist_ok=True) sync.dws_command([sync.DWS,'drive','download','--node',item['fileId'],'--space-id',sync.SPACE,'--output',str(dest),'--format','json']) assert dest.stat().st_size == item['fileSize'], 'Incomplete download' packets.append({'kind':'outbound','date':str(day),'key':key, 'rows':sync.parse(dest,'outbound',str(day),key,products)}) sync.atomic(sync.STATE / ('manual-outbound-'+str(day)+'.json.gz'), gzip.compress(sync.encode(packets))) print('missing sources (skipped):', sorted(sync.SOURCES-files.keys()), flush=True) assert packets and all(p['kind']=='outbound' and p['date']==str(day) and p['key'] in sync.SOURCES for p in packets) print('source rows:', {p['key']:len(p['rows']) for p in packets}, flush=True) changed = sync.apply(conn, packets) for p in packets: assert sync.digest(sync.db_rows(conn,p),'outbound') == sync.digest(p['rows'],'outbound') print(json.dumps({'date':str(day),'changed':changed,'sources':len(packets), 'total_rows':sum(len(p['rows']) for p in packets), 'verified':True,'cache_refreshed':True},ensure_ascii=False), flush=True) finally: conn.close()