Monitor Java Process GC and Restart

#!/usr/bin/perl
#
# Script : monitor_java_process.pl
# Version: 0.1
# Author : sunny kim (sunny@kimster.com)
# Date   : September 22, 2013
#
# Description
# -----------
# Monitors java garbage collection using jstat and restarts it when it's on the verge
# of OOM'ing.  Originally written to handle logstash agent hanging.
#
# Usage
# -----
# perl monitor_java_process.pl --process_name=[string to identify the process from ps]
#                              --init_name=[name of your init script in /etc/init.d/]
#
#

use strict;
use Sys::Syslog;
use Getopt::Long;
use Fcntl ':flock';

INIT
{
        open  *{0} or die "$0:$!";
        flock *{0}, LOCK_EX|LOCK_NB or exit;
}


#--------------------------------------------------------------------
# Global variables
#--------------------------------------------------------------------
my $process_name     = "";
my $init_name        = "";
my $bad1             = 1;
my $bad2             = 1;
my $threshold        = 0.99;
my $duration         = 10;


#--------------------------------------------------------------------
# Syslogging init
#--------------------------------------------------------------------
my $logopt         = "ndelay,nofatal,pid";
my $facility       = "LOG_USER";
my $identity       = $0;
if ($identity =~ /\//) {
        my @split = split(/\//,$identity);
        $identity = pop @split;
}
# Open log connection to syslog
openlog $identity, $logopt, $facility;



#----------------------------------------------------------
# Handle runtime input parameters
#----------------------------------------------------------
my $param = GetOptions (
                                "process_name=s" => \$process_name,
                                "init_name=s"    => \$init_name,
                       );
#----------------------------------------------------------
# get process id
#----------------------------------------------------------
my $pid = `/bin/ps aux | /bin/grep "$process_name" | /bin/grep -v grep | /bin/grep -v monitor_java_process | /usr/bin/awk '{print \$2}'`;
chomp $pid;

#----------------------------------------------------------
# start if it doesn't exist
#----------------------------------------------------------
if (! $pid) {
        my $out = `/usr/sbin/service $init_name start`;
        `/bin/echo "$out" >> /tmp/service.txt`;
        syslog("warning", "started $init_name");
        exit;
}

#----------------------------------------------------------
# run jstat
#----------------------------------------------------------
my @jstat = `/usr/bin/jstat -gc $pid 1000 $duration`;

#----------------------------------------------------------
# evaluate jstat result
#----------------------------------------------------------
FOR:foreach my $jstat (@jstat) {
        next FOR if ($jstat =~/S0C/);
        my @split = split(/\s+/,$jstat);

        my $pct1 = int($split[2]) ? $split[2] / $split[0] : 0;
        my $pct2 = int($split[3]) ? $split[3] / $split[1] : 0;

        if ($pct1 < $threshold) {
                $bad1 = 0;
        }

        if ($pct2 < $threshold) {
                $bad2 = 0;
        }

        print "($split[0]   $split[1]   $split[2]   $split[3])  percentage: ($pct1) ($pct2)\n";
}

#----------------------------------------------------------
# kill and restart if the process is bad
#----------------------------------------------------------
if ($bad1 || $bad2) {
        print "killing process $pid\n";
        `/bin/kill -9 $pid`;
        sleep(10);
        print "restarting $init_name\n";
        my $out = `/usr/sbin/service $init_name start`;
        `/bin/echo "$out" >> /tmp/service.txt`;
        syslog("warning", "restarted $init_name");
}