load_vital.py

from urllib2 import Request, urlopen
from BeautifulSoup import BeautifulSoup, SoupStrainer

dest =  "http://tratu.soha.vn/"
dest = "http://vnexpress.net/"
dest = "http://vnexpress.net/gl/the-gioi/"
dest = "https://www.facebook.com/"
dest = "http://kenh14.vn/"
request  = Request(dest)
response = urlopen(request)

vital_comps = []

def remove_prefix_slash(string):
    """Must do this because we trail all link with /"""
    try:
        if string[0] == '/':
            return string[1:]
        else:
            return string
    except:
        return string

def is_link_alive(link):
    try:
        urlopen(link)
        return True
    except:
        return False

def find_tag_and_check_link(soup, url, tag, before_link):
    #TODO trim url
    for comp in soup.findAll(tag):
        if comp.has_key(before_link):
            string = ''.join(comp[before_link])
            string = remove_prefix_slash(string)

            if string[:7] == "http://" or string[:8] == "https://":
                #external link
                full_link = string
            else:
                #internal link
                full_link = url + string
            print full_link, is_link_alive(full_link)


soup3 = BeautifulSoup(response)
#find all js
find_tag_and_check_link(soup3, dest, 'script', 'src')

#find all css
find_tag_and_check_link(soup3, dest, 'link', 'href')