Guest User

Untitled

a guest
Jan 10th, 2018
130
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.08 KB | None | 0 0
  1. import requests
  2. from bs4 import BeautifulSoup
  3. import mysql.connector
  4.  
  5. url = 'https://www.newegg.com/Video-Cards-Video-Devices/Category/ID-38?Tpk=graphics%20card'
  6.  
  7. source = requests.get(url).text
  8.  
  9. soup = BeautifulSoup(source, 'lxml')
  10.  
  11.  
  12.  
  13. conn = mysql.connector.connect(host='127.0.0.1', user='x', database='scrape',password="x")
  14. cursor = conn.cursor()
  15.  
  16.  
  17.  
  18. item_container = soup.find_all('div', class_='item-container')
  19.  
  20.  
  21. def get_data():
  22. lists = []
  23. for index, item_name in enumerate(item_container):
  24. name = item_name.find_all('a', class_='item-title')[0].text
  25. lists.append({'name': name})
  26. lists[index]['index'] = index
  27.  
  28.  
  29. for index, item_price in enumerate(item_container):
  30. price = item_price.find('li', class_='price-current').find('strong')
  31. if price == None:
  32.  
  33. price == ('Not Available')
  34. lists[index]['price'] = price
  35.  
  36.  
  37. else:
  38.  
  39. price = ('$' + price.text +'.99')
  40. prices = []
  41. lists[index]['price'] = price
  42.  
  43. for index, item_picture in enumerate(item_container):
  44. picture = 'http:' + item_picture.find('img', class_='lazy-img')['data-src']
  45.  
  46. lists[index]['picture'] = picture
  47.  
  48. for index, item_shipping in enumerate(item_container):
  49. shipping = (item_shipping.find('li', class_='price-ship').text).strip()
  50. lists[index]['shipping'] = shipping
  51.  
  52.  
  53. def create_table():
  54.  
  55. val_index = lists[index]['index']
  56. val_name = lists[index]['name']
  57. val_picture = lists[index]['picture']
  58. val_price = lists[index]['price']
  59. val_shipping = lists[index]['shipping']
  60.  
  61.  
  62. add_item = ("INSERT INTO newegg "
  63. "(id, itemname, itempic, itemprice, itemshipping) "
  64. "VALUES (%s, %s, %s, %s, %s)")
  65.  
  66. data_item = (val_index, val_name, val_picture, val_price, val_shipping)
  67.  
  68.  
  69. cursor.execute("DELETE FROM newegg ")
  70. conn.commit()
  71. cursor.execute(add_item, data_item)
  72. conn.commit()
  73.  
  74. cursor.close()
  75. conn.close()
  76.  
  77. create_table();
  78. get_data()
Add Comment
Please, Sign In to add comment