require 'rubygems'
require 'date'
require 'dm-core'
require 'dm-more'

#DataMapper.setup(:default, "sqlite3:///#{Dir.pwd}/test.db")
#DataMapper.setup(:default, 'postgres://localhost/logfile_analysis')
DataMapper.setup(:default, :host => 'localhost', :adapter => 'postgres', :database => 'logfile_analysis_development', :username => 'logfile_analysis', :password => '')

class LogfileLine
  include DataMapper::Resource
  include DataMapper::Validate
  property :id,                     Serial
  property :remote_addr,            IPAddress,  :nullable => false,     :index => true
  property :remote_user,            String,     :nullable => true,      :length => 255
  property :day,                    Integer,    :nullable => false,     :index => true
  property :month,                  Integer,    :nullable => false,     :index => true
  property :year,                   Integer,    :nullable => false,     :index => true
  property :hour,                   Integer,    :nullable => false,     :index => true
  property :min,                    Integer,    :nullable => false
  property :sec,                    Integer,    :nullable => false
  property :request_verb,           String,     :nullable => true,      :length => 7
  property :request_uri,            String,     :nullable => true,      :length => 2048
  property :http_version,           String,     :nullable => true,      :length => 8
  property :status,                 Integer,    :nullable => false,     :index => true
  property :size,                   Integer,    :nullable => false
  property :http_referer,           String,     :nullable => true,      :length => 2048
  property :http_user_agent,        String,     :nullable => true,      :length => 2048
  property :http_x_forwarded_for,   String,     :nullable => true,      :length => 2048
  timestamps  :at
end

DataMapper.auto_migrate!

partial_pattern = /^(?:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))\s+-(.*?)?\s+-\s+\[((\d{1,2})\/(\w+)\/(\d{4}):0?(\d{1,2}):0?(\d{1,2}):0?(\d{1,2})\s+(.*?))\]\s+"(?:-|(GET|HEAD|PUT|POST|DELETE|TRACE|CONNECT)\s+(.*?)\s+(.*?))"\s+(\d{1,3})\s+(\d+)\s+"(.*?)"\s+"(.*?)"\s+"(.*?)"$/i
start_time = Time.now

File.open("nginx.access.log") do |file|
  current_line = 0
  total_lines  = file.lines.count
  file.rewind
  file.each_line { |line|
    line.chomp!
    current_line += 1
    next if line.nil? || line.empty?
    raise line.inspect unless (matches = partial_pattern.match(line))
    remote_addr  =  matches[1]
    remote_user =  matches[2]
    day    = Integer(matches[4])
    str_month  = matches[5]
    i = 0
    month  = Date::ABBR_MONTHNAMES.compact.collect{|name|i = i + 1;[i,name]}.reject{ |i,name| name != str_month }.flatten[0]
    year   = Integer(matches[6])
    hour   = Integer(matches[7])
    min    = Integer(matches[8])
    sec    = Integer(matches[9])
    offset = Integer((Float(matches[10])*0.01).floor)
    request_verb = matches[11]
    request_uri  = matches[12]
    http_version = matches[13]
    status       = Integer(matches[14])
    size         = Integer(matches[15])
    http_referer         = matches[16]
    http_user_agent      = matches[17]
    http_x_forwarded_for =  matches[18]
    logfile_line = LogfileLine.find_or_create({
      :status=>status,
      :request_verb=>request_verb,
      :day=>day,
      :request_uri=>request_uri,
      :remote_addr=>remote_addr,
      :http_version=>http_version,
      :month=>month,
      :sec=>sec,
      :http_referer=>http_referer,
      :min=>min,
      :year=>year,
      :size=>size,
      :hour=>hour,
      :http_user_agent=>http_user_agent,
      :remote_user=>remote_user,
      :http_x_forwarded_for=>http_x_forwarded_for})
    logfile_line.save
    p "id #{logfile_line.id} created - line #{current_line} of #{total_lines} (#{sprintf( "%0.02f", (current_line.to_f / total_lines.to_f) * 100.00 )} % complete)"
    time_now = Time.now
    time_elapsed = time_now - start_time
    lines_left = total_lines - current_line
    lines_per_second = current_line.to_f / time_elapsed
    p "#{time_elapsed} seconds elapsed.  #{sprintf( "%0.02f", lines_per_second)} lines per second.  Estimate finished:  #{Time.now + (lines_left / lines_per_second)}"
    p logfile_line.errors unless logfile_line.errors.empty?
  }
end