import urllib
import urllib.request
import bs4 as bs
import re
import time
import pandas as pd
import requests
import random
import xlsxwriter
import urllib.parse
import urllib.request as rq
import scrapy

class ScrapeCompSource():
    def __init__(self, url):
        self.url = url
        self.soup = self.makeSoup(url)

    def makeSoup(self, url):
        scr = scrapy.Request(url=url)
        r = requests.get(url)
        print(r)


        # req = urllib.request.Request(url)
        # print(req.headers)
        # response = urllib.request.urlopen(req)
        # thepage = response.read()
        # soupData = bs.BeautifulSoup(thepage, "lxml")
        # return soupData

    def getCategories(self):
        mainCategories = []
        for i in self.soup.findAll("div", {"id" : "estores_vensearch"}):
            for j in i.findAll("a"):
                link = re.compile("http://" + j.get("href"))
                if(link not in mainCategories):
                    mainCategories.append(link)

        print(mainCategories)
        print(len(mainCategories))

scrapeComp = ScrapeCompSource("http://www.compsource.com/")