webcrawler

from bs4 import BeautifulSoup
import urllib2

import os

#request url
resp = urllib2.urlopen("https://blockchain.info/blocks")
soup = BeautifulSoup(resp, from_encoding=resp.info().getparam('charset'))

# empty list to build urls
urlList = []
#list of times
timeList = []
#block #
blockList = []
#relay
relaylist =[]

#loop through all links containing a link
for link in soup.find_all('a', href=True):
    #links containing block-index are the hash links
    if 'block-index' in link['href']:

        #build url list of all blocks
        urlList.append("https://blockchain.info"+link['href'])


#print all the links onto console
for i in urlList:

    #prink link
    print "Processing url: " + i + "\n"


    #process data
    resp = urllib2.urlopen(i)
    soup = BeautifulSoup(resp, from_encoding=resp.info().getparam('charset'))

    # NEED DIFFERENT QUERIES TO SEARCH FOR SPECIFIC INFO
    for link in soup.find_all('th', href=False):

        print "\n"
        #split data between spaces
        print link.getText().split(" ")

    #REMOVE THIS
    #USED TO PAUSE THE INFO SO U CAN GET A BETTER VISUAL
    raw_input("Press enter to continue")