Karthik Chikmagalur

#!/usr/bin/python

"""Python script to query Google for definitions of words from the command line

 Usage:
 define [-lr] phrase1 [phrase2 phrase3 ...]
 Options:
         -h : Quick help
         -l : Include URLs to sources of definitions
         -r : Pipe raw HTML to terminal, useful for redirection to browsers
         -f : (Slightly) Fancy output
         -u : Display results in Unicode UTF-8
"""

__author__ = "Karthik Chikmagalur ([email protected])"
__version__ = "$Revision: 0.1 $"
__copyright__ = "(C) Karthik Chikmagalur"
__license_ = "GNU GPL 3"

##############################################################################
# To Do:
# Improve Unicode support
# Port to Windows: minor changes needed
##############################################################################

from httplib import HTTPConnection
from sgmllib import SGMLParser
from optparse import OptionParser
import sys, re, urllib, htmlentitydefs

##############################################################################
#Flags
flag_word={}

# Parse command line options:

cli_parser = OptionParser( usage="usage: %prog [-rlf] phrase1 [phrase2 phrase3 ...]\n%prog --help for help" )
cli_parser.add_option( "-l", "--links", action="store_true", dest="url",
                      default=False, help="Include URLs to sources of definitions" )
cli_parser.add_option( "-r", "--raw", action="store_true", dest="raw",
                      default=False, help="Spout raw HTML, useful for redirection to browsers" )
cli_parser.add_option( "-f", "-F", action="store_true", dest="fancy",
                      default=False, help="Slightly fancier output, makes it easier to read" )
cli_parser.add_option( "-u", "--unicode", action="store_true", dest="unicode",
                      default=False, help="Display results in unicode UTF-8" )

( flag, words ) = cli_parser.parse_args(  )
if not words:
    words = sys.stdin.read( ).strip( ).split( " " )
if not words:
    print "usage: define [-rlf] phrase1 [phrase2 phrase3 ...]\ndefine --help for help"
    sys.exit(  )
##############################################################################
# HTTP Constants

GoogleServer = 'www.google.com'
RequestBase  = '/search?q='
DefineKeyword = 'define: '
# Pretending to be Lynx, simplifies HTML returned by Google (?)
Headers   = { 'User-Agent' : 'Lynx/2.8.6rel.4 libwww-FM/2.14' }

#Example: http://www.google.co.in/search?hl=en&q=define%3A+pine+tree
##############################################################################
# Data Retrieval from Google

if not words:
    sys.exit( "Usage: define [-lrf] phrase1 [phrase2 phrase3 ...]\ndefine --help for help" )

response = {}

try:
    connection = HTTPConnection( GoogleServer )
    for word in words:
        requeststring = RequestBase + urllib.quote_plus( DefineKeyword + word )
        connection.request ( method = 'GET', url = requeststring, headers = Headers )
        response[ word ] = connection.getresponse( ).read( ).strip( )
except Exception, exception:
    sys.exit( 'Unable to make request "http://%s/%s" - %s' % ( GoogleServer, requeststring, exception ) )

##############################################################################
# Regular Expressions

# This regular expression discards the HTML bits not required- which is most of what is returned.
CoarseMatchEx = re.compile ( r'.*on the Web:</p>(?P<def_section>.*?)'
                             r'(<font size=-1><p>|<div style="text-align:center">)',
                             re.DOTALL )

# Matches <a href="some_url">hypertext</a>, that is, any URL bit.
URLMatchEx = re.compile ( r'<a href=.*?</a>', re.MULTILINE )

##############################################################################
# Data processing with Regular expressions defined above

value = {}
for word in words:
    flag_word[ word ] = 1
    coarsematch = CoarseMatchEx.match( response[ word ] )
    if coarsematch:
        value[ word ] = coarsematch.group( 'def_section' )
    else:
        # No match => No definitions found.
        flag_word[ word ] = 0

if not flag.url:
    for word in words:
        # Discard URLs given that definitions are found.
        if flag_word[ word ]: value[ word ] = URLMatchEx.sub( '', value[ word ], 0 )

if flag.raw:
    #Raw output needed, no more processing necessary.
    for word in words:
        if flag_word[ word ]:
            print "<b>%s</b>: " % word.upper(  )
            print value[ word ]
        else:
            print "No definitions found for <b>%s</b><br><br>" % word.upper(  )
    sys.exit(  )

# Done
##############################################################################
# HTML processing and conversion to text

# Class methods for HTML to Text conversion. Most tags will be discarded,
# some replaced by newlines and asterisks, etc.

entitysubs = { 'quot': '"', 'lt' : '<', 'gt' : '>', 'amp' : '&', 'apos' : '\'',
              'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', 'copy':'(C)',
              'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
              'ndash':'-' }
if not flag.unicode:
    charsubs = { 'oelig':'oe', 'aelig':'ae', 'agrave':'a', 'aacute':'a', 'uuml':'u',
                 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', 'egrave':'e',
                 'eacute':'e', 'ecirc':'e', 'euml':'e', 'igrave':'i', 'iacute':'i',
                 'icirc':'i', 'iuml':'i', 'ograve':'o', 'oacute':'o', 'ocirc':'o',
                 'otilde':'o', 'ouml':'o', 'ugrave':'u', 'uacute':'u', 'ucirc':'u' }


class DefinePageHTML2Text( SGMLParser ):
    def reset( self ):
        self.ignore_URL = 0
        self.prefix = ''
        self.suffix = ''
        self.pieces = [  ]
        SGMLParser.reset( self )

    def start_br( self, attrs ):
        self.unknown_starttag( "br", attrs )
        self.pieces.append( "\n" )
        self.prefix= ""
        self.suffix= "\n"

    def start_li( self, attrs ):
        self.unknown_starttag( "li", attrs )
        self.pieces.append( "\n" )
        if flag.fancy: self.prefix = "* "
        self.suffix= ""

    def start_a( self, attrs ):
        self.unknown_starttag( "a", attrs )
        self.ignore_URL = 1
        if attrs[ 0 ][ 0 ]== 'href':
            self.pieces.append( re.match( r'.*&q=(.*?)&usg.*$', attrs[ 0 ][ 1 ] ).group( 1 )+"\n" )

    def handle_charref( self, ref ):
        # Called for each character reference, e.g. for "&#160;", ref will be "160"
        # Convert ref to unicode string and append to output text
        if flag.unicode:
            self.pieces.append( unichr( int( ref ) ).encode( 'utf-8' ) )
        else:
            if ref in htmlentitydefs.codepoint2name.keys( ):
                self.pieces.append( charsubs[ htmlentitydefs.codepoint2name [ref].lower( ) ] )

    def handle_entityref( self, ref ):
        # called for each entity reference, e.g. for "&copy;", ref will be "copy"
        # Replace with closest ASCII equivalent if possible
        if ref in entitysubs.keys( ):
            self.pieces.append( entitysubs[ ref ] )
        else:
            self.pieces.append("&%(ref)s" % locals( ) )
            if htmlentitydefs.entitydefs.has_key( ref ):
                self.pieces.append( ";" )

    def handle_data( self, text ):
        if not self.ignore_URL: self.pieces.extend( [ self.prefix, text, self.suffix ] )
        self.ignore_URL = 0

    def output( self ):
        """Return processed HTML as a single string"""
        return "".join( self.pieces )

parser = DefinePageHTML2Text(  )

#Almost done- Time to feed the parser...
for word in words:
    print str.upper( word )+": ",
    if flag_word[ word ]:
        parser.feed( value[ word ] )
        #...and spit the text out.
        print parser.output(  )
    else:
        #Ouch.
        print "No definitions found for %s\n" % word.upper(  )
    parser.reset(  )

#And that's that.
##############################################################################