Advertisement
Guest User

Untitled

a guest
Jul 16th, 2019
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 5.36 KB | None | 0 0
  1. import argparse
  2. import time
  3. import csv
  4. import os
  5. from concurrent import futures
  6. from multiprocessing import Manager
  7. import enums
  8. from lib.database.database import Database
  9. from lib import models
  10. from async.tasks.product import *
  11.  
  12.  
  13. def update_product(merchant, product_lines, global_images):
  14. for line in product_lines:
  15. while len(line) < 45:
  16. line.append('')
  17.  
  18. for line_data in product_lines:
  19. line_data = line_data[:45]
  20.  
  21. (
  22. product_identifier,
  23. sku_identifier,
  24. hs_code,
  25. product_name,
  26. english_name,
  27. chinese_name,
  28. brand,
  29. material,
  30. target_user_type,
  31. product_category_id,
  32. tags,
  33. length,
  34. width,
  35. height,
  36. weight,
  37. color,
  38. size,
  39. inventory,
  40. enabled,
  41. description,
  42. feature_1,
  43. feature_2,
  44. feature_3,
  45. feature_4,
  46. feature_5,
  47. price,
  48. msrp,
  49. shipping_price,
  50. shipping_time,
  51. has_battery,
  52. has_liquid,
  53. has_powder,
  54. has_paste,
  55. primary_cover_image_url,
  56. secondary_cover_image_url,
  57. product_extra_image_url_1,
  58. product_extra_image_url_2,
  59. product_extra_image_url_3,
  60. product_extra_image_url_4,
  61. product_extra_image_url_5,
  62. product_extra_image_url_6,
  63. product_extra_image_url_7,
  64. product_extra_image_url_8,
  65. product_extra_image_url_9,
  66. product_extra_image_url_10,
  67. ) = line_data
  68.  
  69. tags = [tag.strip() for tag in tags.split(',')]
  70. features = list(filter(bool, [feature_1, feature_2, feature_3, feature_4, feature_5]))
  71. product_extra_image_urls = [
  72. product_extra_image_url_1,
  73. product_extra_image_url_2,
  74. product_extra_image_url_3,
  75. product_extra_image_url_4,
  76. product_extra_image_url_5,
  77. product_extra_image_url_6,
  78. product_extra_image_url_7,
  79. product_extra_image_url_8,
  80. product_extra_image_url_9,
  81. product_extra_image_url_10,
  82. ]
  83.  
  84. print("check cache begin")
  85. images = {}
  86. new_images = []
  87. for extra_url in product_extra_image_urls:
  88. if extra_url in global_images:
  89. print(f'duplicate url:{extra_url}')
  90. images.update({extra_url:global_images[extra_url]})
  91. else:
  92. new_images.append(extra_url)
  93. print("check cache end")
  94.  
  95. if new_images:
  96. new_download_images = batch_generate_image_objects(new_images)
  97. images.update(new_download_images)
  98. global_images.update(new_download_images)
  99.  
  100. extra_images = [
  101. get_image_or_raise(images, url, field_name='Extra Image{} URL'.format(index), required=False)
  102. for index, url in enumerate(product_extra_image_urls)
  103. if url
  104. ]
  105.  
  106. product_identifier = product_identifier.strip()
  107.  
  108. product = Database.get_one_by(
  109. models.Product,
  110. models.Product.merchant_id == merchant.id,
  111. models.Product.identifier == product_identifier,
  112. )
  113. if not product:
  114. raise Exception(f'could not find product({product_identifier})')
  115. else:
  116. product.extra_image_ids = [extra_image.id for extra_image in extra_images]
  117.  
  118. # we just deal one spu
  119. break
  120. Database.commit()
  121.  
  122.  
  123. def batch_update_products(merchant: models.Merchant, product_dict_lines: dict, cache_images: dict):
  124. for key in product_dict_lines:
  125. try:
  126. print(f'deal with {key} begin')
  127. update_product(merchant, product_dict_lines[key], cache_images)
  128. print(f'deal with {key} end')
  129. except Exception:
  130. print(f'deal with {key} failed')
  131.  
  132.  
  133. def main():
  134. parser = argparse.ArgumentParser(description='Import massive goods by csv file')
  135. parser.add_argument('merchant_id', help='Merchant id')
  136. parser.add_argument('sku_csv_folder', help='SKU CSV folder')
  137. args = parser.parse_args()
  138.  
  139. merchant = Database.get_one(models.Merchant, args.merchant_id)
  140. if not merchant:
  141. raise Exception(f'{args.merchant_id} is not exists, please check')
  142.  
  143. # manager = Manager()
  144. # cache_images = manager.dict()
  145.  
  146. cache_images = {}
  147.  
  148. filenames = os.listdir(args.sku_csv_folder)
  149. filenames = sorted(filenames)
  150. for filename in filenames:
  151. file_path = os.path.join(args.sku_csv_folder, filename)
  152. with open(file_path, 'r', errors='ignore') as input_file:
  153. print(f'start deal with {file_path}')
  154. csv_reader = csv.reader(input_file)
  155. failed_lines = []
  156. valid_lines = []
  157.  
  158. a = time.time()
  159. total = 0
  160. product_dict = {}
  161. for line in csv_reader:
  162. if csv_reader.line_num == 1:
  163. continue
  164.  
  165. if line[0] not in product_dict:
  166. product_dict[line[0]] = []
  167. product_dict[line[0]].append(line)
  168.  
  169. batch_update_products(merchant, product_dict, cache_images)
  170.  
  171. b = time.time()
  172. print(b - a)
  173. print(f'end deal with {file_path}')
  174.  
  175.  
  176. if __name__ == "__main__":
  177. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement