281 lines
12 KiB
Python
281 lines
12 KiB
Python
"""Import the WPS cell-image sampling workbook as historical spot records.
|
|
|
|
The workbook's application numbers belong to a different database and are never
|
|
used as local IDs. Historical applicant names are stored as plain text.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import hashlib
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
|
|
NS = {
|
|
'm': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main',
|
|
'r': 'http://schemas.openxmlformats.org/officeDocument/2006/relationships',
|
|
'etc': 'http://www.wps.cn/officeDocument/2017/etCustomData',
|
|
'xdr': 'http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing',
|
|
'a': 'http://schemas.openxmlformats.org/drawingml/2006/main',
|
|
'rel': 'http://schemas.openxmlformats.org/package/2006/relationships',
|
|
}
|
|
EPOCH = dt.datetime(1899, 12, 30)
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
UPLOAD_DIR = ROOT / 'static' / 'uploads' / 'spot_application'
|
|
|
|
|
|
def cell_image_map(archive):
|
|
rel_root = ET.fromstring(archive.read('xl/_rels/cellimages.xml.rels'))
|
|
relationships = {
|
|
rel.attrib['Id']: posixpath.normpath(posixpath.join('xl', rel.attrib['Target']))
|
|
for rel in rel_root
|
|
}
|
|
root = ET.fromstring(archive.read('xl/cellimages.xml'))
|
|
result = {}
|
|
for image in root.findall('etc:cellImage', NS):
|
|
name = image.find('.//xdr:cNvPr', NS).attrib['name']
|
|
rid = image.find('.//a:blip', NS).attrib['{' + NS['r'] + '}embed']
|
|
result[name] = relationships[rid]
|
|
return result
|
|
|
|
|
|
def excel_date(value):
|
|
if not value:
|
|
return None
|
|
if re.fullmatch(r'\d+', value):
|
|
return (EPOCH + dt.timedelta(days=float(value))).date()
|
|
for fmt in ('%Y-%m-%d', '%Y/%m/%d', '%Y%m%d'):
|
|
try:
|
|
return dt.datetime.strptime(value.strip(), fmt).date()
|
|
except ValueError:
|
|
pass
|
|
match = re.fullmatch(r'(\d{1,2})\.(\d{1,2})', value.strip())
|
|
if match:
|
|
month, day = map(int, match.groups())
|
|
year = dt.date.today().year
|
|
try:
|
|
candidate = dt.date(year, month, day)
|
|
if candidate > dt.date.today() + dt.timedelta(days=45):
|
|
candidate = dt.date(year - 1, month, day)
|
|
return candidate
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def arrival_date(value, application_date):
|
|
match = re.fullmatch(r'(\d{1,2})\.(\d{1,2})', (value or '').strip())
|
|
if match and application_date:
|
|
month, day = map(int, match.groups())
|
|
try:
|
|
candidate = dt.date(application_date.year, month, day)
|
|
except ValueError:
|
|
return None
|
|
if candidate < application_date - dt.timedelta(days=45):
|
|
try:
|
|
candidate = dt.date(application_date.year + 1, month, day)
|
|
except ValueError:
|
|
return None
|
|
return candidate
|
|
return excel_date(value)
|
|
|
|
|
|
def read_rows(path):
|
|
with zipfile.ZipFile(path) as archive:
|
|
strings_root = ET.fromstring(archive.read('xl/sharedStrings.xml'))
|
|
strings = [''.join(t.text or '' for t in item.findall('.//m:t', NS))
|
|
for item in strings_root.findall('m:si', NS)]
|
|
images = cell_image_map(archive)
|
|
sheet = ET.fromstring(archive.read('xl/worksheets/sheet1.xml'))
|
|
sheet_rows = sheet.findall('.//m:sheetData/m:row', NS)
|
|
if not sheet_rows:
|
|
raise ValueError('工作表为空')
|
|
|
|
parsed = []
|
|
missing_images = []
|
|
for row in sheet_rows[1:]:
|
|
values = {}
|
|
for cell in row.findall('m:c', NS):
|
|
col = re.match(r'[A-Z]+', cell.attrib['r']).group()
|
|
value_node = cell.find('m:v', NS)
|
|
formula = cell.find('m:f', NS)
|
|
value = value_node.text if value_node is not None else ''
|
|
if cell.attrib.get('t') == 's' and value:
|
|
value = strings[int(value)]
|
|
if col == 'E' and formula is not None:
|
|
match = re.search(r'ID_[A-F0-9]+', formula.text or '')
|
|
value = match.group(0) if match else ''
|
|
values[col] = value or ''
|
|
source_id = values.get('A', '').strip()
|
|
if not source_id:
|
|
continue
|
|
image_id = values.get('E', '').strip()
|
|
image_member = images.get(image_id)
|
|
if image_id and not image_member:
|
|
missing_images.append((source_id, image_id))
|
|
parsed.append((source_id, values, image_member))
|
|
if missing_images:
|
|
raise ValueError(f'有图片引用未找到：{missing_images[:10]}')
|
|
return archive, parsed
|
|
|
|
|
|
def build_application(source_id, values, image_path):
|
|
app_date = excel_date(values.get('B', ''))
|
|
quantity = values.get('G', '').strip()
|
|
if quantity and not re.fullmatch(r'\d+', quantity):
|
|
quantity = '' # Non-numeric notes do not map to the integer quantity field.
|
|
product_name = values.get('D', '').strip()
|
|
if len(product_name) > 300:
|
|
raise ValueError(f'源编号{source_id}产品名称过长')
|
|
applicant_name = values.get('K', '').strip()
|
|
if len(applicant_name) > 200:
|
|
raise ValueError(f'源编号{source_id}申请人姓名过长')
|
|
price_text = values.get('O', '').strip()
|
|
price = price_text if re.fullmatch(r'\d{1,14}(?:\.\d{1,4})?', price_text) else None
|
|
return app_date, {
|
|
'application_date': app_date,
|
|
'department': values.get('I', '').strip(),
|
|
'group_name': values.get('J', '').strip(),
|
|
'applicant_name': applicant_name,
|
|
'brand': values.get('C', '').strip(),
|
|
'product_name': product_name,
|
|
'manufacturer': values.get('L', '').strip(),
|
|
'image_path': image_path,
|
|
'product_link': values.get('H', '').strip(),
|
|
'sample_quantity': int(quantity) if quantity else 0,
|
|
'barcode': values.get('F', '').strip(),
|
|
'recipient_info': values.get('M', '').strip(),
|
|
'status': '通过',
|
|
}, {
|
|
'sample_status': values.get('N', '').strip(),
|
|
'product_unit_price': price,
|
|
'oa_approval_no': values.get('P', '').strip(),
|
|
'express_tracking_no': values.get('Q', '').strip(),
|
|
'arrival_date': arrival_date(values.get('R', ''), app_date),
|
|
'remark': values.get('S', '').strip(),
|
|
'receiving_department': values.get('T', '').strip(),
|
|
'recipient': values.get('U', '').strip(),
|
|
'sample_confirmed': values.get('V', '').strip(),
|
|
# Keep legacy free-text values verbatim, including values outside today's dropdown.
|
|
'sample_confirmation_reason': values.get('W', '').strip(),
|
|
'product_code': values.get('X', '').strip(),
|
|
'final_product_name': values.get('Y', '').strip(),
|
|
'platform': values.get('Z', '').strip(),
|
|
}
|
|
|
|
|
|
def import_rows(workbook, dry_run):
|
|
sys.path.insert(0, str(ROOT))
|
|
if '--local' in sys.argv:
|
|
import test_wsgi # noqa: F401
|
|
from services.db import get_db
|
|
|
|
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
|
with zipfile.ZipFile(workbook) as archive:
|
|
from openpyxl import load_workbook
|
|
from services.spot_confirmation_import import FIELDS
|
|
check_wb = load_workbook(workbook, read_only=True, data_only=False)
|
|
check_sheet = check_wb['采样确认'] if '采样确认' in check_wb.sheetnames else check_wb.worksheets[0]
|
|
headers = [cell.value for cell in next(check_sheet.iter_rows())]
|
|
check_wb.close()
|
|
expected_headers = [field[0] for field in FIELDS]
|
|
if headers != expected_headers:
|
|
raise ValueError(f'模板表头不匹配，预期{expected_headers}，实际{headers}')
|
|
rows = read_rows(workbook)[1]
|
|
prepared = []
|
|
for source_id, values, image_member in rows:
|
|
ext = Path(image_member).suffix.lower() if image_member else ''
|
|
content = archive.read(image_member) if image_member else b''
|
|
digest = hashlib.sha256(content).hexdigest()[:16] if content else ''
|
|
filename = f'history_{source_id}_{digest}{ext}' if ext else ''
|
|
image_path = f'/static/uploads/spot_application/{filename}' if filename else ''
|
|
app_date, app_values, confirmation = build_application(source_id, values, image_path)
|
|
prepared.append((source_id, image_member, filename, app_date, app_values, confirmation))
|
|
|
|
conn = get_db()
|
|
copied = []
|
|
try:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SHOW COLUMNS FROM spot_applications LIKE 'applicant_name'")
|
|
if not cur.fetchone():
|
|
raise RuntimeError('数据库缺少 applicant_name 字段，先部署兼容迁移')
|
|
cur.execute('SELECT id FROM dashboard_users WHERE id=1')
|
|
actor = cur.fetchone()
|
|
if not actor:
|
|
raise RuntimeError('导入审计账号 id=1 不存在')
|
|
if dry_run:
|
|
print(f'预检通过：{len(prepared)}行；图片引用完整；数据库可连接；未写入。')
|
|
conn.rollback()
|
|
return
|
|
|
|
inserted = skipped = 0
|
|
for source_id, image_member, filename, app_date, app_values, confirmation in prepared:
|
|
full_image = UPLOAD_DIR / filename if filename else None
|
|
if full_image and not full_image.exists():
|
|
with open(full_image, 'xb') as f:
|
|
f.write(archive.read(image_member))
|
|
copied.append(full_image)
|
|
elif full_image:
|
|
expected = hashlib.sha256(archive.read(image_member)).hexdigest()[:16]
|
|
actual = hashlib.sha256(full_image.read_bytes()).hexdigest()[:16]
|
|
if actual != expected:
|
|
raise ValueError(f'已有图片校验失败：{filename}')
|
|
if filename:
|
|
cur.execute('SELECT id FROM spot_applications WHERE image_path=%s LIMIT 1',
|
|
(app_values['image_path'],))
|
|
existing = cur.fetchone()
|
|
if existing:
|
|
app_id = existing['id']
|
|
app_update = dict(app_values)
|
|
app_update.pop('applicant_user_id', None)
|
|
app_assign = ','.join(f'`{key}`=%s' for key in app_update)
|
|
cur.execute(f'UPDATE spot_applications SET {app_assign} WHERE id=%s',
|
|
tuple(app_update.values()) + (app_id,))
|
|
cur.execute('INSERT IGNORE INTO spot_confirmations(application_id) VALUES (%s)', (app_id,))
|
|
conf_assign = ','.join(f'`{key}`=%s' for key in confirmation)
|
|
cur.execute(f'UPDATE spot_confirmations SET {conf_assign} WHERE application_id=%s',
|
|
tuple(confirmation.values()) + (app_id,))
|
|
skipped += 1
|
|
continue
|
|
app_values['applicant_user_id'] = 1 # Required audit FK; historical name remains separate text.
|
|
columns = ','.join(f'`{key}`' for key in app_values)
|
|
marks = ','.join(['%s'] * len(app_values))
|
|
cur.execute(f'INSERT INTO spot_applications ({columns}) VALUES ({marks})',
|
|
tuple(app_values.values()))
|
|
app_id = cur.lastrowid
|
|
cfields = {'application_id': app_id, **confirmation}
|
|
ccolumns = ','.join(f'`{key}`' for key in cfields)
|
|
cmarks = ','.join(['%s'] * len(cfields))
|
|
cur.execute(f'INSERT INTO spot_confirmations ({ccolumns}) VALUES ({cmarks})',
|
|
tuple(cfields.values()))
|
|
inserted += 1
|
|
conn.commit()
|
|
print(f'导入完成：新增{inserted}条，按图片路径跳过已有{skipped}条。')
|
|
except Exception:
|
|
conn.rollback()
|
|
for path in copied:
|
|
try:
|
|
path.unlink()
|
|
except OSError:
|
|
pass
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('workbook')
|
|
parser.add_argument('--local', action='store_true')
|
|
parser.add_argument('--dry-run', action='store_true')
|
|
args = parser.parse_args()
|
|
import_rows(args.workbook, args.dry_run)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|