Python/ HTMLParser example for searchengine.py

""" I haven't added this to searchengine.py yet, but I find this parses more forgivingly than BeautifulSoup. """


from HTMLParser import HTMLParser
import urllib2 as urllib2
from string import punctuation

class MyHTMLParser(HTMLParser):

    count=0
    head='active'
    result=''

    def handle_starttag(self, tag, attrs):
        if self.head=='passive':
            if tag=='script':
                self.count=0
        if tag=='title':
            self.count=1

    def handle_endtag(self, tag):
        if tag=="head":
            self.count=1
            self.head='passive'
        if self.head=='passive':
            if tag=='script':
                self.count=1
        if tag=='title':
            self.count=0

    def handle_data(self, data):
        if self.count > 0:
            self.result = self.result + data

def crawl(page):
    try:
        c = urllib2.urlopen(page)
        parser = MyHTMLParser()
        parser.feed(c.read())
        return parser.result
    except:
        print 'Couldn\'t parse %s' % page

if __name__ == "__main__":
    while 1+1==2:
        page = str(raw_input('Enter URL to crawl >'))
        result = crawl(page)
        result = result.replace('\n',' ').replace('\t',' ').replace('\r',' ').lower()
        result1 = result.split(' ')
        for a in range(len(result1)):
            if result1[a] in punctuation: result1[a] = ""
            if result1[a][-1:] in punctuation:
                result1[a] = result1[a].replace('.', ' ')[:-1].lower()
        for b in result1:
            if b != '': print b,' ',