ScrapeHoodCal.py

#!/usr/bin/python

from bs4 import BeautifulSoup
import pprint
import re
import StringIO
import sys
import urllib2

MONTHS = ['January','February','March','April','May','June',\
'July','August','September','October','November','December']
DAYS_OF_WEEK = ['Monday','Tuesday','Wednesday','Thursday',\
'Friday','Saturday','Sunday']
REGEX_YEAR = '^([A-Z][a-z]+\s?\w+)\s+(\d{4})$'
REGEX_DAY = '^([A-Z][a-z]{2,5}day)\s+(\d+)$'

'''
Retrieve the text from a meta tag from the Hood College website.
'''
def retrieveText(url):
    data = urllib2.urlopen(url)       # type 'instance
    html = data.read()                # type 'str'
    soup = BeautifulSoup(html)        # class BeautifulSoup.BeautifulSoup
    resultSet = soup.findAll('meta',
        attrs={"name":"Description"}) # class BeautifulSoup.ResultSet
    tag = resultSet[0]                # class BeautifulSoup.Tag
    ucode = tag['content']            # type 'unicode'

    return ucode.encode('utf-8').strip()

'''
Create a dictionary given a block of text.
'''
def createDictionary(text, debug):
    def console(frmt, *args, **kwargs):
        if debug:
            print(frmt.format(*args, **kwargs))

    aDict = {}
    currYear, currMonth, currDay = None, None, None
    i, ioStr = 0, StringIO.StringIO(text)
    for rawLine in ioStr:
        procLine = rawLine.strip().rstrip()
        if procLine:
            ### Test Year
            m = re.search(REGEX_YEAR, procLine)
            if m is not None:
                console('{:03d} Found Year -- {:s}', i, procLine)
                year = m.group(2)
                #year = procLine
                if not currYear == year:
                    currYear, currMonth, currDay = year, None, None
                if currYear not in aDict:
                    aDict[currYear] = {}
                continue

            ### Test Month
            if currYear and procLine in MONTHS:
                month = MONTHS.index(procLine)
                if not month == currMonth:
                    currMonth, currDay = month, None
                console('{:03d} Found Month -- {:s}', i, procLine)
                if currMonth is not None:
                    if currMonth not in aDict[currYear]:
                        aDict[currYear][currMonth] = {}
                    continue

            ### Test Day
            if currMonth:
                m = re.search(REGEX_DAY, procLine)
                if m is not None:
                    dayOfWeek, day = m.group(1), '{:02d}'.format(int(m.group(2)))
                    if not currDay == day:
                        currDay = day
                    console('{:03d} Found Day -- {:s}', i, procLine)
                    if currDay not in aDict[currYear][currMonth]:
                        aDict[currYear][currMonth][currDay] = {}
                    continue

            #''' If the dictionary is ready, append the lines. ''''
            if currDay:
                if not aDict[currYear][currMonth][currDay]:
                    aDict[currYear][currMonth][currDay] = ''
                aDict[currYear][currMonth][currDay] += procLine + ' '

            i += 1

    return aDict

'''
Write a dictionary object to a file with a specified indent level.
'''
def writeToFile(filename, aDict, indentLevel):
    with open(filename, 'wt') as out:
        pp = pprint.PrettyPrinter(indent=indentLevel,stream=out)
        pp.pprint(aDict)

'''
Main procedure for this script.
'''
if __name__ == '__main__':
    if len(sys.argv) > 2:
        print('Usage: ' + sys.argv[0] + ' {-d?}')
        exit(0)

    debug = False
    if len(sys.argv) == 2 and sys.argv[1] == '-d':
        debug = True

    filename = 'output.txt'
    html = retrieveText('http://www.hood.edu/calendar2013')
    aDict = createDictionary(html, debug)
    writeToFile(filename, aDict, 2)
    print('Wrote JSON object to: ' + filename)