Advertisement
Guest User

Untitled

a guest
Nov 24th, 2017
80
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 11.84 KB | None | 0 0
  1. import os
  2. import requests
  3. import datetime
  4. from xml.etree import ElementTree
  5. from django.core.management.base import BaseCommand, CommandError
  6. from django.conf import settings
  7. from django.utils import timezone
  8. from parsers.models import CategoryTng, ProductTng, ParserLog
  9. from settings.models import Pricing
  10. from goods.models import Provider
  11.  
  12. url = "http://www.tngtoys.ru/Go/ProductData?branch_id=138"
  13. url_rossosh = "http://www.tngtoys.ru/prices/PriceRossosh.xml"
  14.  
  15.  
  16. class Command(BaseCommand):
  17.  
  18.     def get_barcodes(self, tree):
  19.         barcodes = []
  20.  
  21.         for rossosh in tree.iter('product'):
  22.             rossosh_barcode = rossosh.find('ShtrihKod').text
  23.  
  24.             barcodes.append(rossosh_barcode)
  25.  
  26.         return barcodes
  27.  
  28.     def download_file(self, url):
  29.         local_filename = os.path.join(settings.ROOT_DIR, 'ostatki_tng.xml')
  30.         r = requests.get(url, stream=True)
  31.         with open(local_filename, 'wb') as f:
  32.             for chunk in r.iter_content(chunk_size=1024):
  33.                 if chunk:
  34.                     f.write(chunk)
  35.  
  36.         return local_filename
  37.  
  38.     def get_categories(self, tree):
  39.         categories = []
  40.  
  41.         for cat in tree.iter('category'):
  42.             cat_id = cat.get('id')
  43.             cat_name = cat.text
  44.  
  45.             categories.append({
  46.                 'id': cat_id,
  47.                 'name': cat_name
  48.             })
  49.         return categories
  50.  
  51.     def get_offers(self, tree):
  52.         products = []
  53.  
  54.         for offer in tree.iter('offer'):
  55.             offer_id = offer.attrib['id']
  56.             offer_price = offer.find('price').text
  57.             offer_currency = offer.find('currencyId').text
  58.             offer_category = offer.find('categoryId').text
  59.             offer_picture = offer.find('picture').text
  60.             offer_name = offer.find('name').text
  61.             offer_description = offer.find('description').text
  62.             offer_ref = offer.find('ref').text
  63.             offer_code = offer.find('code').text
  64.             offer_manufacturer = offer.find('manufacturer').text
  65.             offer_weight = offer.find('weight').text
  66.             offer_size_x = offer.find('size_x').text
  67.             offer_size_y = offer.find('size_y').text
  68.             offer_size_z = offer.find('size_z').text
  69.             offer_volume = offer.find('volume').text
  70.             offer_new = offer.find('new').text
  71.             offer_hit = offer.find('hit').text
  72.             offer_action = offer.find('action').text
  73.             offer_sale = offer.find('sale').text
  74.             offer_in_box = offer.find('in_box').text
  75.             offer_in_carton = offer.find('in_carton').text
  76.             offer_stock = offer.find('stock').text
  77.             offer_nds = offer.find('nds').text
  78.             offer_metka = offer.find('metka').text
  79.             offer_shtrihkod = offer.find('ShtrihKod').text
  80.             offer_country = offer.find('country').text
  81.             offer_nomengroup = offer.find('NomenGroup').text
  82.  
  83.             products.append({
  84.                 'offer_id': offer_id,
  85.                 'price': offer_price,
  86.                 'price_opt': offer_price,
  87.                 'currency': offer_currency,
  88.                 'category': offer_category,
  89.                 'picture': offer_picture,
  90.                 'name': offer_name,
  91.                 'description': offer_description,
  92.                 'ref': offer_ref,
  93.                 'code': offer_code,
  94.                 'manufacturer': offer_manufacturer,
  95.                 'weight': offer_weight,
  96.                 'size_x': offer_size_x,
  97.                 'size_y': offer_size_y,
  98.                 'size_z': offer_size_z,
  99.                 'volume': offer_volume,
  100.                 'new': offer_new,
  101.                 'hit': offer_hit,
  102.                 'action': offer_action,
  103.                 'sale': offer_sale,
  104.                 'in_box': offer_in_box,
  105.                 'in_carton': offer_in_carton,
  106.                 'stock': offer_stock,
  107.                 'nds': offer_nds,
  108.                 'metka': offer_metka,
  109.                 'shtrihkod': offer_shtrihkod,
  110.                 'country': offer_country,
  111.                 'nomengroup': offer_nomengroup,
  112.             })
  113.         return products
  114.  
  115.     def get_price_mrc(self, current_price):
  116.         tng_provider = Provider.objects.filter(code='TG').first()
  117.         current_pricing = Pricing.objects.filter(price_from__lt=current_price, price_to__gt=current_price,
  118.                                                  provider__id=tng_provider.id).first()
  119.         if current_pricing:
  120.             current_coef = current_pricing.coef
  121.             price_mrc = float(current_price) * current_coef
  122.         else:
  123.             price_mrc = None
  124.         if price_mrc:
  125.             return int(price_mrc)
  126.         else:
  127.             return 0
  128.  
  129.     def handle(self, *args, **options):
  130.         parser_log = ParserLog()
  131.         parser_log.name = "Запустили скрипт tng_parser"
  132.         parser_log.content = "Инициализация"
  133.         parser_log.save()
  134.         response = requests.get(url, verify=False)
  135.         tree = ElementTree.fromstring(response.content)
  136.  
  137.         response_rossosh = requests.get(url_rossosh, verify=False)
  138.         tree_rossosh = ElementTree.fromstring(response_rossosh.content)
  139.         _root, created = CategoryTng.objects.get_or_create(name='_root')
  140.         zero_category, z_creat = CategoryTng.objects.get_or_create(name='Без категории', category_id=0, parent=_root)
  141.  
  142.         print(datetime.datetime.now())
  143.         # self.download_file(url)
  144.         # print('Скачивание завершено')
  145.         products = self.get_offers(tree)
  146.         categories = self.get_categories(tree)
  147.         barcodes = self.get_barcodes(tree_rossosh)
  148.         # brands = []
  149.  
  150.         # Обнуляем остатки
  151.         ProductTng.objects.all().update(count_moscow=0, count_spb=0)
  152.  
  153.         for cat in categories:
  154.             ext_category = CategoryTng.objects.filter(category_id=cat['id']).first()
  155.             if not ext_category:
  156.                 category = CategoryTng()
  157.                 category.category_id = cat['id']
  158.                 category.name = cat['name']
  159.                 category.parent = _root
  160.                 category.save()
  161.             else:
  162.                 pass
  163.  
  164.         for prod in products:
  165.             current_category = int(prod['category'])
  166.             current_barcode = prod['shtrihkod']
  167.  
  168.             category = CategoryTng.objects.filter(category_id=current_category).first()
  169.             product_tng = ProductTng.objects.filter(barcode=prod['shtrihkod']).first()
  170.  
  171.             if current_barcode in barcodes:
  172.                 # brands.append(prod['manufacturer'])
  173.                 try:
  174.                     stock_int = int(prod['stock'])
  175.                     if not product_tng:
  176.                         product = ProductTng()
  177.                         product.offer_id = prod['offer_id']
  178.                         product.price = prod['price']
  179.                         product.price_opt = prod['price']
  180.                         product.currency = prod['currency']
  181.                         product.image = prod['picture']
  182.                         product.name = prod['name']
  183.                         product.description = prod['description']
  184.                         product.artikul = prod['ref']
  185.                         product.code = prod['code']
  186.                         product.manufacturer = prod['manufacturer']
  187.                         product.weight = prod['weight']
  188.                         product.size_x = prod['size_x']
  189.                         product.size_y = prod['size_y']
  190.                         product.size_z = prod['size_z']
  191.                         product.volume = prod['volume']
  192.                         product.new = prod['new']
  193.                         product.hit = prod['hit']
  194.                         product.action = prod['action']
  195.                         product.sale = prod['sale']
  196.                         product.in_box = prod['in_box']
  197.                         product.in_carton = prod['in_carton']
  198.                         # product.stock = prod['stock'] if prod['stock'] else 0
  199.                         product.count_moscow = stock_int if stock_int > 0 else 0
  200.                         product.nds = prod['nds']
  201.                         product.metka = prod['metka']
  202.                         product.barcode = prod['shtrihkod']
  203.                         product.country = prod['country']
  204.                         product.nomengroup = prod['nomengroup']
  205.  
  206.                         current_price = prod['price']
  207.                         if current_price:
  208.                             price_mrc = round(self.get_price_mrc(current_price), -1)
  209.                         if price_mrc:
  210.                             product.price_mrc = price_mrc
  211.  
  212.                         product.category = category if category else zero_category
  213.                         product.save()
  214.  
  215.                         product.product_id = 't' + str(product.id)
  216.                         product.save()
  217.                     else:
  218.                         product_tng.offer_id = prod['offer_id']
  219.                         product_tng.price = prod['price']
  220.                         product_tng.price_opt = prod['price']
  221.                         product_tng.currency = prod['currency']
  222.                         product_tng.image = prod['picture']
  223.                         product_tng.name = prod['name']
  224.                         product_tng.description = prod['description']
  225.                         product_tng.artikul = prod['ref']
  226.                         product_tng.code = prod['code']
  227.                         product_tng.manufacturer = prod['manufacturer']
  228.                         product_tng.weight = prod['weight']
  229.                         product_tng.size_x = prod['size_x']
  230.                         product_tng.size_y = prod['size_y']
  231.                         product_tng.size_z = prod['size_z']
  232.                         product_tng.volume = prod['volume']
  233.                         product_tng.new = prod['new']
  234.                         product_tng.hit = prod['hit']
  235.                         product_tng.action = prod['action']
  236.                         product_tng.sale = prod['sale']
  237.                         product_tng.in_box = prod['in_box']
  238.                         product_tng.in_carton = prod['in_carton']
  239.                         product_tng.count_moscow = stock_int if stock_int > 0 else 0
  240.                         product_tng.nds = prod['nds']
  241.                         product_tng.metka = prod['metka']
  242.                         product_tng.barcode = prod['shtrihkod']
  243.                         product_tng.country = prod['country']
  244.                         product_tng.nomengroup = prod['nomengroup']
  245.  
  246.                         current_price = prod['price']
  247.                         if current_price:
  248.                             price_mrc = round(self.get_price_mrc(current_price), -1)
  249.  
  250.                         if price_mrc:
  251.                             product_tng.price_mrc = price_mrc
  252.                         product_tng.category = category if category else zero_category
  253.                         product_tng.save()
  254.  
  255.                         product_tng.product_id = 't' + str(product_tng.id)
  256.                         product_tng.save()
  257.                 except Exception as e:
  258.                     print(str(e))
  259.                     print(prod['stock'])
  260.                     print(prod['shtrihkod'])
  261.                     print(type(product_tng))
  262.  
  263.         empty_mrc = ProductTng.objects.filter(price_mrc=None).all()
  264.         for e in empty_mrc:
  265.             e.price_mrc = round(self.get_price_mrc(e.price_opt), -1)
  266.             print('пустая МРЦ заполнена' + str(e.id))
  267.             e.save()
  268.  
  269.         print(datetime.datetime.now())
  270.         print("Скрипт tng_parser завершил свою работу")
  271.         parser_log.date_submitted = timezone.now()
  272.         parser_log.content = 'Скрипт tng_parser завершил свою работу'
  273.         parser_log.save()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement