List-urls.py

#!/usr/bin/python
"""Extract list of URLs in a web page

This program is part of "Dive Into Python", a free Python book for
experienced programmers.  Visit http://diveintopython.org/ for the
latest version.
"""

__author__ = "Mark Pilgrim (mark@diveintopython.org)"
__version__ = "$Revision: 1.2 $"
__date__ = "$Date: 2004/05/05 21:57:19 $"
__copyright__ = "Copyright (c) 2001 Mark Pilgrim"
__license__ = "Python"

from sgmllib import SGMLParser
import sys

if len(sys.argv) != 2:
	print "\n\n+++++++++++++++++++++++++++++++++++++++++++++++++++++"
	print "Extract links form webpage - v.0.1            "
	print "+++++++++++++++++++++++++++++++++++++++++++++++++++++"
	print "\nUsage : ./list-urls.py <web-page>            "
	print "Eg: ./list-urls.py http://www.whoppix.net          "
	print "\n+++++++++++++++++++++++++++++++++++++++++++++++++++++"
	sys.exit(1)



class URLLister(SGMLParser):
	def reset(self):
		SGMLParser.reset(self)
		self.urls = []

	def start_a(self, attrs):
		href = [v for k, v in attrs if k=='href']
		if href:
			self.urls.extend(href)

if __name__ == "__main__":

	import urllib
	print "\n##########################################################"
	print "# 							#"
	print "#	     Extract URLS from a web page		#"
	print "#		muts@whitehat.co.il			#"
	print "# 							#"
	print "##########################################################\n"
	link = sys.argv[1]
	try:
		usock = urllib.urlopen(link)
		parser = URLLister()
		parser.feed(usock.read())
		parser.close()
		usock.close()
		for url in parser.urls: print url
	except:
		print "Could not reach "+ sys.argv[1]+ " !"
		print "Did you remember to put an http:// before the domain name?"