Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import argparse
- import time
- import csv
- import os
- from concurrent import futures
- from multiprocessing import Manager
- import enums
- from lib.database.database import Database
- from lib import models
- from async.tasks.product import *
- def update_product(merchant, product_lines, global_images):
- for line in product_lines:
- while len(line) < 45:
- line.append('')
- for line_data in product_lines:
- line_data = line_data[:45]
- (
- product_identifier,
- sku_identifier,
- hs_code,
- product_name,
- english_name,
- chinese_name,
- brand,
- material,
- target_user_type,
- product_category_id,
- tags,
- length,
- width,
- height,
- weight,
- color,
- size,
- inventory,
- enabled,
- description,
- feature_1,
- feature_2,
- feature_3,
- feature_4,
- feature_5,
- price,
- msrp,
- shipping_price,
- shipping_time,
- has_battery,
- has_liquid,
- has_powder,
- has_paste,
- primary_cover_image_url,
- secondary_cover_image_url,
- product_extra_image_url_1,
- product_extra_image_url_2,
- product_extra_image_url_3,
- product_extra_image_url_4,
- product_extra_image_url_5,
- product_extra_image_url_6,
- product_extra_image_url_7,
- product_extra_image_url_8,
- product_extra_image_url_9,
- product_extra_image_url_10,
- ) = line_data
- tags = [tag.strip() for tag in tags.split(',')]
- features = list(filter(bool, [feature_1, feature_2, feature_3, feature_4, feature_5]))
- product_extra_image_urls = [
- product_extra_image_url_1,
- product_extra_image_url_2,
- product_extra_image_url_3,
- product_extra_image_url_4,
- product_extra_image_url_5,
- product_extra_image_url_6,
- product_extra_image_url_7,
- product_extra_image_url_8,
- product_extra_image_url_9,
- product_extra_image_url_10,
- ]
- print("check cache begin")
- images = {}
- new_images = []
- for extra_url in product_extra_image_urls:
- if extra_url in global_images:
- print(f'duplicate url:{extra_url}')
- images.update({extra_url:global_images[extra_url]})
- else:
- new_images.append(extra_url)
- print("check cache end")
- if new_images:
- new_download_images = batch_generate_image_objects(new_images)
- images.update(new_download_images)
- global_images.update(new_download_images)
- extra_images = [
- get_image_or_raise(images, url, field_name='Extra Image{} URL'.format(index), required=False)
- for index, url in enumerate(product_extra_image_urls)
- if url
- ]
- product_identifier = product_identifier.strip()
- product = Database.get_one_by(
- models.Product,
- models.Product.merchant_id == merchant.id,
- models.Product.identifier == product_identifier,
- )
- if not product:
- raise Exception(f'could not find product({product_identifier})')
- else:
- product.extra_image_ids = [extra_image.id for extra_image in extra_images]
- # we just deal one spu
- break
- Database.commit()
- def batch_update_products(merchant: models.Merchant, product_dict_lines: dict, cache_images: dict):
- for key in product_dict_lines:
- try:
- print(f'deal with {key} begin')
- update_product(merchant, product_dict_lines[key], cache_images)
- print(f'deal with {key} end')
- except Exception:
- print(f'deal with {key} failed')
- def main():
- parser = argparse.ArgumentParser(description='Import massive goods by csv file')
- parser.add_argument('merchant_id', help='Merchant id')
- parser.add_argument('sku_csv_folder', help='SKU CSV folder')
- args = parser.parse_args()
- merchant = Database.get_one(models.Merchant, args.merchant_id)
- if not merchant:
- raise Exception(f'{args.merchant_id} is not exists, please check')
- # manager = Manager()
- # cache_images = manager.dict()
- cache_images = {}
- filenames = os.listdir(args.sku_csv_folder)
- filenames = sorted(filenames)
- for filename in filenames:
- file_path = os.path.join(args.sku_csv_folder, filename)
- with open(file_path, 'r', errors='ignore') as input_file:
- print(f'start deal with {file_path}')
- csv_reader = csv.reader(input_file)
- failed_lines = []
- valid_lines = []
- a = time.time()
- total = 0
- product_dict = {}
- for line in csv_reader:
- if csv_reader.line_num == 1:
- continue
- if line[0] not in product_dict:
- product_dict[line[0]] = []
- product_dict[line[0]].append(line)
- batch_update_products(merchant, product_dict, cache_images)
- b = time.time()
- print(b - a)
- print(f'end deal with {file_path}')
- if __name__ == "__main__":
- main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement