"""Import the WPS cell-image sampling workbook as historical spot records. The workbook's application numbers belong to a different database and are never used as local IDs. Historical applicant names are stored as plain text. """ import argparse import datetime as dt import hashlib import os import posixpath import re import sys import xml.etree.ElementTree as ET import zipfile from pathlib import Path NS = { 'm': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main', 'r': 'http://schemas.openxmlformats.org/officeDocument/2006/relationships', 'etc': 'http://www.wps.cn/officeDocument/2017/etCustomData', 'xdr': 'http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing', 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', 'rel': 'http://schemas.openxmlformats.org/package/2006/relationships', } EPOCH = dt.datetime(1899, 12, 30) ROOT = Path(__file__).resolve().parents[1] UPLOAD_DIR = ROOT / 'static' / 'uploads' / 'spot_application' def cell_image_map(archive): rel_root = ET.fromstring(archive.read('xl/_rels/cellimages.xml.rels')) relationships = { rel.attrib['Id']: posixpath.normpath(posixpath.join('xl', rel.attrib['Target'])) for rel in rel_root } root = ET.fromstring(archive.read('xl/cellimages.xml')) result = {} for image in root.findall('etc:cellImage', NS): name = image.find('.//xdr:cNvPr', NS).attrib['name'] rid = image.find('.//a:blip', NS).attrib['{' + NS['r'] + '}embed'] result[name] = relationships[rid] return result def excel_date(value): if not value: return None if re.fullmatch(r'\d+', value): return (EPOCH + dt.timedelta(days=float(value))).date() for fmt in ('%Y-%m-%d', '%Y/%m/%d', '%Y%m%d'): try: return dt.datetime.strptime(value.strip(), fmt).date() except ValueError: pass match = re.fullmatch(r'(\d{1,2})\.(\d{1,2})', value.strip()) if match: month, day = map(int, match.groups()) year = dt.date.today().year try: candidate = dt.date(year, month, day) if candidate > dt.date.today() + dt.timedelta(days=45): candidate = dt.date(year - 1, month, day) return candidate except ValueError: return None return None def arrival_date(value, application_date): match = re.fullmatch(r'(\d{1,2})\.(\d{1,2})', (value or '').strip()) if match and application_date: month, day = map(int, match.groups()) try: candidate = dt.date(application_date.year, month, day) except ValueError: return None if candidate < application_date - dt.timedelta(days=45): try: candidate = dt.date(application_date.year + 1, month, day) except ValueError: return None return candidate return excel_date(value) def read_rows(path): with zipfile.ZipFile(path) as archive: strings_root = ET.fromstring(archive.read('xl/sharedStrings.xml')) strings = [''.join(t.text or '' for t in item.findall('.//m:t', NS)) for item in strings_root.findall('m:si', NS)] images = cell_image_map(archive) sheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml')) sheet_rows = sheet.findall('.//m:sheetData/m:row', NS) if not sheet_rows: raise ValueError('工作表为空') parsed = [] missing_images = [] for row in sheet_rows[1:]: values = {} for cell in row.findall('m:c', NS): col = re.match(r'[A-Z]+', cell.attrib['r']).group() value_node = cell.find('m:v', NS) formula = cell.find('m:f', NS) value = value_node.text if value_node is not None else '' if cell.attrib.get('t') == 's' and value: value = strings[int(value)] if col == 'E' and formula is not None: match = re.search(r'ID_[A-F0-9]+', formula.text or '') value = match.group(0) if match else '' values[col] = value or '' source_id = values.get('A', '').strip() if not source_id: continue image_id = values.get('E', '').strip() image_member = images.get(image_id) if image_id and not image_member: missing_images.append((source_id, image_id)) parsed.append((source_id, values, image_member)) if missing_images: raise ValueError(f'有图片引用未找到：{missing_images[:10]}') return archive, parsed def build_application(source_id, values, image_path): app_date = excel_date(values.get('B', '')) quantity = values.get('G', '').strip() if quantity and not re.fullmatch(r'\d+', quantity): quantity = '' # Non-numeric notes do not map to the integer quantity field. product_name = values.get('D', '').strip() if len(product_name) > 300: raise ValueError(f'源编号{source_id}产品名称过长') applicant_name = values.get('K', '').strip() if len(applicant_name) > 200: raise ValueError(f'源编号{source_id}申请人姓名过长') price_text = values.get('O', '').strip() price = price_text if re.fullmatch(r'\d{1,14}(?:\.\d{1,4})?', price_text) else None return app_date, { 'application_date': app_date, 'department': values.get('I', '').strip(), 'group_name': values.get('J', '').strip(), 'applicant_name': applicant_name, 'brand': values.get('C', '').strip(), 'product_name': product_name, 'manufacturer': values.get('L', '').strip(), 'image_path': image_path, 'product_link': values.get('H', '').strip(), 'sample_quantity': int(quantity) if quantity else 0, 'barcode': values.get('F', '').strip(), 'recipient_info': values.get('M', '').strip(), 'status': '通过', }, { 'sample_status': values.get('N', '').strip(), 'product_unit_price': price, 'oa_approval_no': values.get('P', '').strip(), 'express_tracking_no': values.get('Q', '').strip(), 'arrival_date': arrival_date(values.get('R', ''), app_date), 'remark': values.get('S', '').strip(), 'receiving_department': values.get('T', '').strip(), 'recipient': values.get('U', '').strip(), 'sample_confirmed': values.get('V', '').strip(), # Keep legacy free-text values verbatim, including values outside today's dropdown. 'sample_confirmation_reason': values.get('W', '').strip(), 'product_code': values.get('X', '').strip(), 'final_product_name': values.get('Y', '').strip(), 'platform': values.get('Z', '').strip(), } def import_rows(workbook, dry_run): sys.path.insert(0, str(ROOT)) if '--local' in sys.argv: import test_wsgi # noqa: F401 from services.db import get_db UPLOAD_DIR.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(workbook) as archive: from openpyxl import load_workbook from services.spot_confirmation_import import FIELDS check_wb = load_workbook(workbook, read_only=True, data_only=False) check_sheet = check_wb['采样确认'] if '采样确认' in check_wb.sheetnames else check_wb.worksheets[0] headers = [cell.value for cell in next(check_sheet.iter_rows())] check_wb.close() expected_headers = [field[0] for field in FIELDS] if headers != expected_headers: raise ValueError(f'模板表头不匹配，预期{expected_headers}，实际{headers}') rows = read_rows(workbook)[1] prepared = [] for source_id, values, image_member in rows: ext = Path(image_member).suffix.lower() if image_member else '' content = archive.read(image_member) if image_member else b'' digest = hashlib.sha256(content).hexdigest()[:16] if content else '' filename = f'history_{source_id}_{digest}{ext}' if ext else '' image_path = f'/static/uploads/spot_application/{filename}' if filename else '' app_date, app_values, confirmation = build_application(source_id, values, image_path) prepared.append((source_id, image_member, filename, app_date, app_values, confirmation)) conn = get_db() copied = [] try: with conn.cursor() as cur: cur.execute("SHOW COLUMNS FROM spot_applications LIKE 'applicant_name'") if not cur.fetchone(): raise RuntimeError('数据库缺少 applicant_name 字段，先部署兼容迁移') cur.execute('SELECT id FROM dashboard_users WHERE id=1') actor = cur.fetchone() if not actor: raise RuntimeError('导入审计账号 id=1 不存在') if dry_run: print(f'预检通过：{len(prepared)}行；图片引用完整；数据库可连接；未写入。') conn.rollback() return inserted = skipped = 0 for source_id, image_member, filename, app_date, app_values, confirmation in prepared: full_image = UPLOAD_DIR / filename if filename else None if full_image and not full_image.exists(): with open(full_image, 'xb') as f: f.write(archive.read(image_member)) copied.append(full_image) elif full_image: expected = hashlib.sha256(archive.read(image_member)).hexdigest()[:16] actual = hashlib.sha256(full_image.read_bytes()).hexdigest()[:16] if actual != expected: raise ValueError(f'已有图片校验失败：{filename}') if filename: cur.execute('SELECT id FROM spot_applications WHERE image_path=%s LIMIT 1', (app_values['image_path'],)) existing = cur.fetchone() if existing: app_id = existing['id'] app_update = dict(app_values) app_update.pop('applicant_user_id', None) app_assign = ','.join(f'`{key}`=%s' for key in app_update) cur.execute(f'UPDATE spot_applications SET {app_assign} WHERE id=%s', tuple(app_update.values()) + (app_id,)) cur.execute('INSERT IGNORE INTO spot_confirmations(application_id) VALUES (%s)', (app_id,)) conf_assign = ','.join(f'`{key}`=%s' for key in confirmation) cur.execute(f'UPDATE spot_confirmations SET {conf_assign} WHERE application_id=%s', tuple(confirmation.values()) + (app_id,)) skipped += 1 continue app_values['applicant_user_id'] = 1 # Required audit FK; historical name remains separate text. columns = ','.join(f'`{key}`' for key in app_values) marks = ','.join(['%s'] * len(app_values)) cur.execute(f'INSERT INTO spot_applications ({columns}) VALUES ({marks})', tuple(app_values.values())) app_id = cur.lastrowid cfields = {'application_id': app_id, **confirmation} ccolumns = ','.join(f'`{key}`' for key in cfields) cmarks = ','.join(['%s'] * len(cfields)) cur.execute(f'INSERT INTO spot_confirmations ({ccolumns}) VALUES ({cmarks})', tuple(cfields.values())) inserted += 1 conn.commit() print(f'导入完成：新增{inserted}条，按图片路径跳过已有{skipped}条。') except Exception: conn.rollback() for path in copied: try: path.unlink() except OSError: pass raise finally: conn.close() def main(): parser = argparse.ArgumentParser() parser.add_argument('workbook') parser.add_argument('--local', action='store_true') parser.add_argument('--dry-run', action='store_true') args = parser.parse_args() import_rows(args.workbook, args.dry_run) if __name__ == '__main__': main()