#!/usr/bin/ruby


require 'tmpdir'
require 'open3'
require 'socket'


##----------------------------------------------------------------------
## CONSTANTS
##----------------------------------------------------------------------
ICOTS_ROOT = '/var/www/rails/icots'
PID_DIR = "#{ICOTS_ROOT}/shared/pids"

SURVEILLANCE_THRESHOLD = 500
PREJUDICE_THRESHOLD = 625
TERMINATE_THRESHOLD = 750

UNDER_SURVEILLANCE = Hash.new


##----------------------------------------------------------------------
## methods
##----------------------------------------------------------------------

def check_and_possibly_create_tmp_animal_control
  tacd = Dir.tmpdir + File::SEPARATOR + 'animal_control'
  Dir.mkdir( tacd ) unless File.exist?( tacd )
end

def read_mongrel_pid( mongrel_pid_filename )

  mongrel_pid = String.new
  File.open( mongrel_pid_filename ) { |f| f.read( nil, mongrel_pid )  }
  puts "mongrel_pid = '#{mongrel_pid}'" if $DEBUG

  mongrel_pid.to_i

end


def under_surveillance?( mongrel_pid )
  us = UNDER_SURVEILLANCE.has_key?( mongrel_pid )
  puts "under_surveillance? = '#{us}'" if $DEBUG
  us
end


def over_surveillance_threshold?( mongrel_pid )

  vsz_in_mb = `ps -o vsz= #{mongrel_pid}`.chomp.to_i / 1024
  puts "vsz_in_mb = '#{vsz_in_mb}'" if $DEBUG
  puts "SURVEILLANCE_THRESHOLD = '#{SURVEILLANCE_THRESHOLD}'" if $DEBUG

  ost = vsz_in_mb > SURVEILLANCE_THRESHOLD
  puts "over_surveillance_threshold? = '#{ost}'" if $DEBUG

  ost

end


def over_terminate_threshold?( mongrel_pid )

  vsz_in_mb = `ps -o vsz= #{mongrel_pid}`.chomp.to_i / 1024
  puts "vsz_in_mb = '#{vsz_in_mb}'" if $DEBUG
  puts "TERMINATE_THRESHOLD = '#{TERMINATE_THRESHOLD}'" if $DEBUG

  ott = vsz_in_mb > TERMINATE_THRESHOLD
  puts "over_terminate_threshold = '#{ott}'" if $DEBUG

  ott

end


def over_prejudice_threshold?( mongrel_pid )

  vsz_in_mb = `ps -o vsz= #{mongrel_pid}`.chomp.to_i / 1024
  puts "vsz_in_mb = '#{vsz_in_mb}'" if $DEBUG
  puts "PREJUDICE_THRESHOLD = '#{PREJUDICE_THRESHOLD}'" if $DEBUG

  opt = vsz_in_mb > PREJUDICE_THRESHOLD
  puts "over_prejudice_threshold = '#{opt}'" if $DEBUG

  opt

end


def should_surveil?( mongrel_pid )
  v = (! under_surveillance?(mongrel_pid)) && over_surveillance_threshold?( mongrel_pid )
  puts "should_surveil? = '#{v}'" if $DEBUG
  v
end


def should_terminate?( mongrel_pid )
  v = under_surveillance?(mongrel_pid) && over_terminate_threshold?( mongrel_pid )
    puts "should_terminate? = '#{v}'" if $DEBUG
  v
end


def should_terminate_with_prejudice?( mongrel_pid )
  v = (! under_surveillance?(mongrel_pid)) && over_prejudice_threshold?( mongrel_pid )
  puts "should_terminate_with_prejudice? = '#{v}'" if $DEBUG
  v
end

def tmp_filename( mongrel_pid_filename )
  tf = String.new
  tf << Dir.tmpdir
  tf << File::SEPARATOR
  tf << 'animal_control'
  tf << File::SEPARATOR
  tf << Socket.gethostname
  tf << '_'
  tf << File.basename(mongrel_pid_filename)[0,12].gsub('.','_')
  tf << '_'
  tf << Time.now.strftime('%Y%m%d_%H%M%S')
  tf << '.'
  tf << 'strace'
  puts "tmp_filename = '#{tf}'" if $DEBUG
  tf
end

def start_surveillance( mongrel_pid_filename, mongrel_port, mongrel_pid )

  strace_filename = tmp_filename( mongrel_pid_filename )
  cpid = fork { exec("sudo strace -p #{mongrel_pid} -o #{strace_filename}; gzip #{strace_filename}; echo 'see attachment' | mail -s '#{File.basename(strace_filename)}.gz' -a #{strace_filename}.gz jheath@appriss.com wsheldahl@appriss.com") }
  Process.detach( cpid )

  UNDER_SURVEILLANCE[ mongrel_pid ] = true
  `echo "surveilling mongrel #{Socket.gethostname}:#{mongrel_port}" | mail -s "surveilling mongrel #{Socket.gethostname}:#{mongrel_port}" jheath@appriss.com wsheldahl@appriss.com aocsystems2@appriss.com`
end


def call_start
  sleep 5
  `sudo -u appserv mongrel_rails cluster::start -C /var/www/rails/icots/current/config/mongrel/prod/icots.yml`
end


def terminate_mongrel( mongrel_pid_filename, mongrel_port, mongrel_pid )

  Process.kill('SIGKILL', mongrel_pid )

  UNDER_SURVEILLANCE.delete( mongrel_pid )

  File.delete( mongrel_pid_filename )

  `echo "terminated mongrel #{Socket.gethostname}:#{mongrel_port}" | mail -s "terminated mongrel #{Socket.gethostname}:#{mongrel_port}" jheath@appriss.com wsheldahl@appriss.com aocsystems2@appriss.com`
  call_start

end


def terminate_runaway_mongrel( mongrel_pid_filename, mongrel_port, mongrel_pid )
  Process.kill( 'SIGKILL', mongrel_pid )
  `echo "terminated runaway mongrel not under surveillance #{Socket.gethostname}:#{mongrel_port}" | mail -s "terminated runaway mongrel not under surveillance #{Socket.gethostname}:#{mongrel_port}" jheath@app
  File.delete( mongrel_pid_filename )
  call_start
end


##----------------------------------------------------------------------
## MAIN
##----------------------------------------------------------------------

puts "animal_control.rb started at #{Time.now}"

check_and_possibly_create_tmp_animal_control

loop do

  Dir.glob("#{PID_DIR}/*.pid").each do |mongrel_pid_filename|

    mongrel_pid = read_mongrel_pid( mongrel_pid_filename )
    mongrel_port = File.basename(mongrel_pid_filename)[8,4].to_i

    start_surveillance( mongrel_pid_filename, mongrel_port, mongrel_pid ) if should_surveil?( mongrel_pid )
    terminate_mongrel( mongrel_pid_filename, mongrel_port, mongrel_pid ) if should_terminate?( mongrel_pid )
    terminate_runaway_mongrel( mongrel_pid_filename, mongrel_port, mongrel_pid ) if should_terminate_with_prejudice?( mongrel_pid )

  end

  sleep 1

end