Robots_file v0.1

#!/usr/bin/python
# script coded by datahack from pentest101.blogspot.com
# Robots_files very smart script to find allowed directories and sitemaps in robots.txt
# we are not responsible for any bad usage
# have fun

import urllib
import re
import sys


def usage():
        print "#### script coded by Pentest101 Team ####"
        print "#### informations gathering          ####"
        print "[+] usage : ./robots_file.py [options] [site] (without /)"
        print "[+] options:"
        print "         -A : Allowed"
        print "         -D : Disalowed"
        print "         -S : Sitemap"
        sys.exit()


def allow(site):
    urllib.urlretrieve(site + '/robots.txt','/tmp/robots.txt')
    f = open('/tmp/robots.txt')
        print "---Allowed Directorys : ",site
        print ""
    for dir in f.readlines():
                if 'Allow' in dir:
            dir = re.sub('\n$','',dir)
            dir_2 = re.search(r'/\S+',dir)
            print site+dir_2.group()
    f.close()
    print "coded by Pentest101 team"

def disallow(site):
        urllib.urlretrieve(site + '/robots.txt','/tmp/robots.txt')
        f = open('/tmp/robots.txt')
        print "---Disallowed Directorys : ",site
        print ""
        for dir in f.readlines():
                if 'Disallow' in dir:
                        dir = re.sub('\n$','',dir)
                        dir_2 = re.search(r'/\S+',dir)
                        print site+dir_2.group()
        f.close()
        print "coded by Pentest101 team"

def sitemap(site):
    urllib.urlretrieve(site + '/robots.txt','/tmp/robots.txt')
        f = open('/tmp/robots.txt')
        print "---Sitemap : ",site
        print ""
        for dir in f.readlines():
                if 'Sitemap' in dir:
                        dir = re.sub('\n$','',dir)
                        dir_2 = re.search(r'http://\S+',dir)
            print dir_2.group()
    f.close()
    print "coded by Pentest101 team"
def main():
    try:
        if sys.argv[1] == "-A":
            allow(sys.argv[2])
        elif sys.argv[1] == "-D":
                        disallow(sys.argv[2])
        elif sys.argv[1] == "-S":
                        sitemap(sys.argv[2])
        else:
            usage()
    except:
        usage()
# good to go
main()