import requests, pprint, json
from bs4 import BeautifulSoup
from peewee import *

# MySQL Connect
db = MySQLDatabase('steam', user='root', password='toor', host='localhost', charset='utf8')

# Model
class BaseModel(Model):
    class Meta:
        database = db

class Games(BaseModel):
    name = CharField(max_length=255)

class Images(BaseModel):
    small   = CharField(max_length=255)

class Items(BaseModel):
    name    = CharField(max_length=255)
    game_id = IntegerField()
    image   = IntegerField()
    link    = CharField(max_length=255)
    price   = CharField(max_length=255)

# Create the tables.
db.create_tables([Games, Images, Items], safe=True)

pp = pprint.PrettyPrinter(indent=4)

for page in xrange(1, 24617):
    r = requests.get('https://steamcommunity.com/market/search/render/?query=&start=%s&count=10&search_descriptions=0&sort_column=popular&sort_dir=desc' % page * 10)

    print ' [!] Page %s of %s' % (page, '24617')

    soup = BeautifulSoup(json.loads(r.text)['results_html'], 'html.parser')

    items = soup.find_all('a', class_ = 'market_listing_row_link')

    for item in items:
        data = {}

        data['name']  = item.find('span', class_ = 'market_listing_item_name').get_text()
        data['game']  = item.find('span', class_ = 'market_listing_game_name').get_text()
        data['link']  = item['href']
        data['img']   = item.find('img', class_ = 'market_listing_item_img')['src']
        data['price'] = item.find('span', class_ = 'sale_price').get_text()

        # game
        game, created = Games.get_or_create(name=data['game'])
        game = game or created

        if not created == None:
            print ' [!] New game added "%s"' % data['game']

        # image
        image = Images()
        image.small = data['img']
        image.save()

        # item
        newItem = Items()
        newItem.name = data['name']
        newItem.game_id = game.id
        newItem.image = image.id
        newItem.link = data['link']
        newItem.price = data['price']
        newItem.save()

        print ' [~] New item "%s" - %s' % (data['name'], data['price'])