#!/usr/bin/python

import re, argparse, os

def show_sorted(dictionary):
    for entry in sorted(dictionary, key=dictionary.get, reverse=True):
        if options.has_key('minimum') == False or dictionary[entry] > int(options['minimum']):
            print str(dictionary[entry]).rjust(total_digits), entry
    print ""

parser = argparse.ArgumentParser(description='Parse a lighttpd access log.')

parser.add_argument('logfiles', metavar='logfile', type=str, nargs='+',
                   help='path(s) of the logfile(s)')

parser.add_argument('-e', '--extensions', dest='extensions', action='store',
                   help='specify a comma-separated list of extensions to ignore during parsing')

parser.add_argument('-m', '--minimum', dest='minimum', action='store',
                   help='the counting threshold that has to be exceeded to display the entry')

args = parser.parse_args()
options = vars(args)

referers = {}
days = {}
hosts = {}
files = {}
urls = {}
extensions = {}

total_digits = 10
current_lines = 0

try:
    ignore_extensions = options['extensions'].split(',')
except AttributeError:
    ignore_extensions = []

for logpath in options['logfiles']:
    try:
        log = open(logpath, 'r')

        for line in log:
            try:
                ip, hostname, dash, datetime, timezone, method, uri, version, status, size, referer, useragent = line.split(' ', 11)
                datetime = datetime[1:]
                date = datetime.split(':')[0]
                timezone = timezone[:-1]
                method = method[1:]
                version = version[:-1]
                useragent = useragent[1:-2]
                referer = referer[1:-1]
                filename = uri.split('?')[0]
                extension = os.path.splitext(filename)[1][1:]

                if extension not in ignore_extensions:
                    if hostname not in hosts:
                        hosts[hostname] = 0

                    if referer not in referers:
                        referers[referer] = 0

                    if date not in days:
                        days[date] = 0

                    if filename not in files:
                        files[filename] = 0

                    if uri not in urls:
                        urls[uri] = 0

                    if extension not in extensions:
                        extensions[extension] = 0

                    hosts[hostname] += 1
                    referers[referer] += 1
                    days[date] += 1
                    files[filename] += 1
                    urls[uri] += 1
                    extensions[extension] += 1
            except ValueError:
                print "Corrupt log line at line %d, contents: %s" % (current_lines + 1, line[:-1])

            current_lines += 1

            if current_lines % 1000 == 0:
                print "Processed %d lines." % current_lines
    except IOError:
        print "Could not find file %s, ignored entry." % logpath

print "Top days:"
show_sorted(days)
print ""

print "Top requested hostnames:"
show_sorted(hosts)
print ""

print "Top files:"
show_sorted(files)
print ""

print "Top extensions:"
show_sorted(extensions)
print ""

print "Top referers:"
show_sorted(referers)
print ""

print "Top URLs:"
show_sorted(urls)
print ""