217 lines
11 KiB
Python
217 lines
11 KiB
Python
"""Import a historical workbook with external named images, with a portable checked manifest."""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import shutil
|
||
import sys
|
||
from collections import defaultdict
|
||
from datetime import date, datetime
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
|
||
def prepare(workbook, images, destination):
|
||
import openpyxl
|
||
from services.spot_confirmation_import import FIELDS, _date, _integer
|
||
|
||
def verify_image(path):
|
||
head = path.read_bytes()[:16]
|
||
valid = (head.startswith(b'\x89PNG\r\n\x1a\n') or head.startswith(b'\xff\xd8\xff') or
|
||
head.startswith((b'GIF87a', b'GIF89a', b'BM', b'II*\x00', b'MM\x00*')) or
|
||
(head.startswith(b'RIFF') and head[8:12] == b'WEBP'))
|
||
assert valid, f'无法识别图片格式：{path.name}'
|
||
index = defaultdict(list)
|
||
for path in Path(images).rglob('*'):
|
||
if path.is_file():
|
||
index[path.name].append(path)
|
||
wb = openpyxl.load_workbook(workbook, read_only=True, data_only=False)
|
||
sheet = wb['采样确认']
|
||
rows = sheet.iter_rows()
|
||
headers = [cell.value for cell in next(rows)]
|
||
assert headers == [f[0] for f in FIELDS], '模板表头不匹配'
|
||
destination = Path(destination)
|
||
destination.mkdir(parents=True, exist_ok=True)
|
||
(destination / 'images').mkdir(exist_ok=True)
|
||
prepared, seen = [], set()
|
||
for number, cells in enumerate(rows, 2):
|
||
if not any(c.value is not None for c in cells):
|
||
continue
|
||
assert not any(c.data_type == 'f' for c in cells), f'第{number}行含公式'
|
||
application, confirmation = {}, {}
|
||
source_id = str(cells[0].value).strip()
|
||
assert source_id not in seen, f'重复源编号{source_id}'
|
||
seen.add(source_id)
|
||
image_name = str(cells[4].value or '').strip()
|
||
assert len(index[image_name]) == 1, f'图片未唯一匹配：{image_name}'
|
||
path = index[image_name][0]
|
||
verify_image(path)
|
||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||
filename = 'named_' + source_id + '_' + digest[:16] + path.suffix.lower()
|
||
target = destination / 'images' / filename
|
||
if not target.exists():
|
||
shutil.copyfile(path, target)
|
||
for cell, (label, key, table, limit) in zip(cells, FIELDS):
|
||
if table is None:
|
||
continue
|
||
value = cell.value
|
||
text = str(value).strip() if value is not None else ''
|
||
if key == 'image_path':
|
||
value = '/static/uploads/spot_application/' + filename
|
||
elif key in ('application_date', 'arrival_date'):
|
||
value = _date(value, label).isoformat() if text else None
|
||
elif key == 'sample_quantity':
|
||
value = _integer(value, label) if text else 0
|
||
elif key == 'product_unit_price':
|
||
value = text or None
|
||
else:
|
||
value = text
|
||
if limit:
|
||
assert (len(str(value).encode()) if limit == 65535 else len(str(value))) <= limit, f'第{number}行{label}过长'
|
||
(application if table == 'application' else confirmation)[key] = value
|
||
assert application['product_name'], f'第{number}行缺少产品名称'
|
||
application['status'] = '通过'
|
||
prepared.append(dict(source_id=source_id, row=number, image_name=image_name,
|
||
sha256=digest, application=application, confirmation=confirmation))
|
||
wb.close()
|
||
manifest = dict(workbook_sha256=hashlib.sha256(Path(workbook).read_bytes()).hexdigest(), records=prepared)
|
||
(destination / 'manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
print('Prepared records:', len(prepared), 'unique images:', len(list((destination / 'images').iterdir())))
|
||
|
||
|
||
def verify(bundle, local):
|
||
if local:
|
||
import test_wsgi
|
||
from services.db import get_db
|
||
bundle = Path(bundle)
|
||
records = json.loads((bundle / 'manifest.json').read_text())['records']
|
||
mappings = json.loads((bundle / ('result-local.json' if local else 'result-cloud.json')).read_text())
|
||
by_source = {m['source_id']: m['application_id'] for m in mappings}
|
||
assert len(by_source) == len(records)
|
||
conn = get_db()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
for record in records:
|
||
pk = by_source[record['source_id']]
|
||
cur.execute('SELECT * FROM spot_applications WHERE id=%s', (pk,))
|
||
application = cur.fetchone()
|
||
cur.execute('SELECT * FROM spot_confirmations WHERE application_id=%s', (pk,))
|
||
confirmation = cur.fetchone()
|
||
for expected, actual in ((record['application'], application), (record['confirmation'], confirmation)):
|
||
assert actual is not None, pk
|
||
for key, value in expected.items():
|
||
actual_value = actual[key]
|
||
if isinstance(actual_value, (date, datetime)):
|
||
actual_value = actual_value.isoformat()
|
||
assert actual_value == value, (pk, key, actual_value, value)
|
||
image = ROOT / record['application']['image_path'].lstrip('/')
|
||
assert hashlib.sha256(image.read_bytes()).hexdigest() == record['sha256'], pk
|
||
print('Verified all fields and image hashes for', len(records), 'records')
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def run(bundle, local, apply, start_id=None):
|
||
if local:
|
||
from test_wsgi import app
|
||
else:
|
||
from app import app
|
||
from services.db import get_db, add_operation_log
|
||
from services.cache import bump_data_version
|
||
bundle = Path(bundle)
|
||
manifest = json.loads((bundle / 'manifest.json').read_text())
|
||
records = manifest['records']
|
||
conn = get_db()
|
||
mappings, new = [], []
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute('SELECT id,status,product_name,application_date,applicant_name,image_path FROM spot_applications')
|
||
existing = cur.fetchall()
|
||
for record in records:
|
||
values = record['application']
|
||
filename = Path(values['image_path']).name
|
||
source = bundle / 'images' / filename
|
||
assert hashlib.sha256(source.read_bytes()).hexdigest() == record['sha256'], filename
|
||
matches = [r for r in existing if r['image_path'] == values['image_path'] and
|
||
r['product_name'] == values['product_name'] and
|
||
str(r['application_date'] or '') == str(values['application_date'] or '') and
|
||
(r['applicant_name'] or '') == values['applicant_name']]
|
||
# Never overwrite an unrelated historical entry based on a source ID.
|
||
overlaps = [r for r in existing if r['product_name'] == values['product_name'] and
|
||
str(r['application_date'] or '') == str(values['application_date'] or '') and
|
||
(r['applicant_name'] or '') == values['applicant_name'] and
|
||
not str(r['image_path'] or '').startswith('/static/uploads/spot_application/named_')]
|
||
assert not overlaps, f"源编号{record['source_id']}与现有记录冲突：{overlaps}"
|
||
assert len(matches) <= 1, f"源编号{record['source_id']}有多条匹配"
|
||
mappings.append(dict(source_id=record['source_id'], application_id=matches[0]['id'] if matches else None))
|
||
if not matches:
|
||
new.append((record, mappings[-1]))
|
||
print('Preflight:', len(records), 'new:', len(new), 'already imported:', len(records)-len(new), flush=True)
|
||
if not apply:
|
||
conn.rollback()
|
||
return
|
||
cur.execute('SELECT id FROM dashboard_users WHERE id=1')
|
||
assert cur.fetchone(), 'Missing audit actor'
|
||
target_ids = ({record['source_id']: start_id + index for index, record in enumerate(records)}
|
||
if start_id is not None else {})
|
||
if target_ids:
|
||
occupied = [row for row in existing if row['id'] in set(target_ids.values()) and
|
||
not str(row.get('image_path') or '').startswith('/static/uploads/spot_application/named_')]
|
||
next_safe = max([row['id'] for row in existing] + list(target_ids.values())) + 1000
|
||
cur.execute('SET FOREIGN_KEY_CHECKS=0')
|
||
try:
|
||
for row in occupied:
|
||
cur.execute('UPDATE spot_confirmations SET application_id=%s WHERE application_id=%s', (next_safe, row['id']))
|
||
cur.execute('UPDATE spot_applications SET id=%s WHERE id=%s', (next_safe, row['id']))
|
||
next_safe += 1
|
||
finally:
|
||
cur.execute('SET FOREIGN_KEY_CHECKS=1')
|
||
upload = ROOT / 'static/uploads/spot_application'
|
||
upload.mkdir(parents=True, exist_ok=True)
|
||
for record, mapping in new:
|
||
values = dict(record['application'], applicant_user_id=1)
|
||
if start_id is not None:
|
||
values['id'] = target_ids[record['source_id']]
|
||
filename = Path(values['image_path']).name
|
||
target = upload / filename
|
||
if target.exists():
|
||
assert hashlib.sha256(target.read_bytes()).hexdigest() == record['sha256']
|
||
else:
|
||
shutil.copyfile(bundle / 'images' / filename, target)
|
||
cols = ','.join('`'+k+'`' for k in values)
|
||
cur.execute(f"INSERT INTO spot_applications ({cols}) VALUES ({','.join(['%s']*len(values))})", tuple(values.values()))
|
||
mapping['application_id'] = values.get('id') or cur.lastrowid
|
||
confirmation = dict(record['confirmation'], application_id=mapping['application_id'])
|
||
cols = ','.join('`'+k+'`' for k in confirmation)
|
||
cur.execute(f"INSERT INTO spot_confirmations ({cols}) VALUES ({','.join(['%s']*len(confirmation))})", tuple(confirmation.values()))
|
||
conn.commit()
|
||
(bundle / ('result-local.json' if local else 'result-cloud.json')).write_text(json.dumps(mappings, ensure_ascii=False, indent=2))
|
||
with app.test_request_context():
|
||
bump_data_version()
|
||
add_operation_log('现货采样确认导入', f"文件名匹配图片批量导入，新增{len(new)}条，来源文件SHA256={manifest['workbook_sha256']}", user_id=1, page='现货采样确认')
|
||
print('Committed:', len(new), 'images and source ID mappings verified', flush=True)
|
||
except Exception:
|
||
conn.rollback()
|
||
raise
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
if __name__ == '__main__':
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument('--workbook')
|
||
parser.add_argument('--images')
|
||
parser.add_argument('--bundle', required=True)
|
||
parser.add_argument('--local', action='store_true')
|
||
parser.add_argument('--apply', action='store_true')
|
||
parser.add_argument('--verify', action='store_true')
|
||
parser.add_argument('--start-id', type=int)
|
||
args = parser.parse_args()
|
||
if args.workbook:
|
||
prepare(args.workbook, args.images, args.bundle)
|
||
elif args.verify:
|
||
verify(args.bundle, args.local)
|
||
else:
|
||
run(args.bundle, args.local, args.apply, args.start_id)
|