SHARE
TWEET

Untitled

a guest Jul 16th, 2019 71 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import argparse
  2. import time
  3. import csv
  4. import os
  5. from concurrent import futures
  6. from multiprocessing import Manager
  7. import enums
  8. from lib.database.database import Database
  9. from lib import models
  10. from async.tasks.product import *
  11.  
  12.  
  13. def update_product(merchant, product_lines, global_images):
  14.     for line in product_lines:
  15.         while len(line) < 45:
  16.             line.append('')
  17.  
  18.     for line_data in product_lines:
  19.         line_data = line_data[:45]
  20.  
  21.         (
  22.             product_identifier,
  23.             sku_identifier,
  24.             hs_code,
  25.             product_name,
  26.             english_name,
  27.             chinese_name,
  28.             brand,
  29.             material,
  30.             target_user_type,
  31.             product_category_id,
  32.             tags,
  33.             length,
  34.             width,
  35.             height,
  36.             weight,
  37.             color,
  38.             size,
  39.             inventory,
  40.             enabled,
  41.             description,
  42.             feature_1,
  43.             feature_2,
  44.             feature_3,
  45.             feature_4,
  46.             feature_5,
  47.             price,
  48.             msrp,
  49.             shipping_price,
  50.             shipping_time,
  51.             has_battery,
  52.             has_liquid,
  53.             has_powder,
  54.             has_paste,
  55.             primary_cover_image_url,
  56.             secondary_cover_image_url,
  57.             product_extra_image_url_1,
  58.             product_extra_image_url_2,
  59.             product_extra_image_url_3,
  60.             product_extra_image_url_4,
  61.             product_extra_image_url_5,
  62.             product_extra_image_url_6,
  63.             product_extra_image_url_7,
  64.             product_extra_image_url_8,
  65.             product_extra_image_url_9,
  66.             product_extra_image_url_10,
  67.         ) = line_data
  68.  
  69.         tags = [tag.strip() for tag in tags.split(',')]
  70.         features = list(filter(bool, [feature_1, feature_2, feature_3, feature_4, feature_5]))
  71.         product_extra_image_urls = [
  72.             product_extra_image_url_1,
  73.             product_extra_image_url_2,
  74.             product_extra_image_url_3,
  75.             product_extra_image_url_4,
  76.             product_extra_image_url_5,
  77.             product_extra_image_url_6,
  78.             product_extra_image_url_7,
  79.             product_extra_image_url_8,
  80.             product_extra_image_url_9,
  81.             product_extra_image_url_10,
  82.         ]
  83.  
  84.         print("check cache begin")
  85.         images = {}
  86.         new_images = []
  87.         for extra_url in product_extra_image_urls:
  88.             if extra_url in global_images:
  89.                 print(f'duplicate url:{extra_url}')
  90.                 images.update({extra_url:global_images[extra_url]})
  91.             else:
  92.                 new_images.append(extra_url)
  93.         print("check cache end")
  94.    
  95.         if new_images:
  96.             new_download_images = batch_generate_image_objects(new_images)
  97.             images.update(new_download_images)
  98.             global_images.update(new_download_images)
  99.  
  100.         extra_images = [
  101.             get_image_or_raise(images, url, field_name='Extra Image{} URL'.format(index), required=False)
  102.             for index, url in enumerate(product_extra_image_urls)
  103.             if url
  104.         ]
  105.  
  106.         product_identifier = product_identifier.strip()
  107.  
  108.         product = Database.get_one_by(
  109.             models.Product,
  110.             models.Product.merchant_id == merchant.id,
  111.             models.Product.identifier == product_identifier,
  112.         )
  113.         if not product:
  114.             raise Exception(f'could not find product({product_identifier})')
  115.         else:
  116.             product.extra_image_ids = [extra_image.id for extra_image in extra_images]
  117.  
  118.         # we just deal one spu
  119.         break
  120.     Database.commit()
  121.  
  122.  
  123. def batch_update_products(merchant: models.Merchant, product_dict_lines: dict, cache_images: dict):
  124.     for key in product_dict_lines:
  125.         try:
  126.             print(f'deal with {key} begin')
  127.             update_product(merchant, product_dict_lines[key], cache_images)
  128.             print(f'deal with {key} end')
  129.         except Exception:
  130.             print(f'deal with {key} failed')
  131.  
  132.  
  133. def main():
  134.     parser = argparse.ArgumentParser(description='Import massive goods by csv file')
  135.     parser.add_argument('merchant_id', help='Merchant id')
  136.     parser.add_argument('sku_csv_folder', help='SKU CSV folder')
  137.     args = parser.parse_args()
  138.  
  139.     merchant = Database.get_one(models.Merchant, args.merchant_id)
  140.     if not merchant:
  141.         raise Exception(f'{args.merchant_id} is not exists, please check')
  142.  
  143.     # manager = Manager()
  144.     # cache_images = manager.dict()
  145.  
  146.     cache_images = {}
  147.  
  148.     filenames = os.listdir(args.sku_csv_folder)
  149.     filenames = sorted(filenames)
  150.     for filename in filenames:
  151.         file_path = os.path.join(args.sku_csv_folder, filename)
  152.         with open(file_path, 'r', errors='ignore') as input_file:
  153.             print(f'start deal with {file_path}')
  154.             csv_reader = csv.reader(input_file)
  155.             failed_lines = []
  156.             valid_lines = []
  157.  
  158.             a = time.time()
  159.             total = 0
  160.             product_dict = {}
  161.             for line in csv_reader:
  162.                 if csv_reader.line_num == 1:
  163.                     continue
  164.  
  165.                 if line[0] not in product_dict:
  166.                     product_dict[line[0]] = []
  167.                 product_dict[line[0]].append(line)
  168.  
  169.             batch_update_products(merchant, product_dict, cache_images)
  170.  
  171.             b = time.time()
  172.             print(b - a)
  173.             print(f'end deal with {file_path}')
  174.  
  175.  
  176. if __name__ == "__main__":
  177.     main()
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top