#!/usr/bin/perl

# author: Oleg Korchagin
# license: GPL v2 or newer
# Скрипт для xymon. Мониторит состояние агентов puppet.
# Запускаться должен на хосте с puppet master'ом.
# http://www.linux.org.ru/forum/admin/8588141



use strict;
use Storable;
use Data::Dumper;
use Time::Local;
use Getopt::Std;
use Getopt::Long;

my $store_file='/usr/lib/xymon/client/data/check_puppet_log.store';
my $puppet_log_file='/var/log/messages';

# $data - data to load from and safe to $store_file
# structure:
# {
#   pos - last position in $puppet_log_file
#   agents  => {
#       $agent  => {
#           last_manifest_require   - last request of manifest, in unixtime
#           last_compile_time   - last catalog coptile time for this node
#           last_apply_time - last manifest apply time on this node
#       }
#   }
# }
my $data;

my $warn_last_require = 3600;
my $err_last_require = 5400;

my $warn_compile = 10;
my $err_compile = 60;

my $warn_apply = 200;
my $err_apply = 600;

my $host = $ENV{'MACHINE'} ? $ENV{'MACHINE'} : 'vml-puppet';
my $test = "puppet_agents";

my $version = '0.1a';

# - file
# - position
sub go_last_pos($$) {
    my ( $FILE, $pos ) = @_;

    # check size
    seek( $FILE, 0, 2); # go to end of file

    seek( $FILE, tell($FILE) < $pos ? 0 : $pos, 0); # go to last position, if no logrotate. Or to start of file
}

# convert time to unixtime. string example:
#  Dec  9 03:33:01 vml-puppet-test rsyslogd:
sub str2ut($) {
    my $time_shift = 0;
    my %m=(
        "Jan" => 0, "Feb"=>1,"Mar"=>2,"Apr"=>3,"May"=>4,"Jun"=>5,"Jul"=>6, "Aug"=>7,"Sep"=>8,"Oct"=>9,"Nov"=>10,"Dec"=>11,
        "Янв"   => 0, "Фев" => 1, "Мар" => 2, "Апр" => 3, "Май" => 4, "Июн" => 5, "Июл" => 6, "Авг" => 7, "Сен" => 8, "Окт" => 9, "Ноя" => 10, "Дек" => 11
    );
    $_[0] =~ /^(\S{3,6})\s+([0-9]{1,2})\s+([0-9]{2}):([0-9]{2}):([0-9]{2})\s/ || print "can't decode date$_[0]";
    my ( $year, $month, $day, $hour, $min, $sec ) = (0,$m{$1},$2,$3,$4,$5);

    my ( $now_year, $now_month ) = ( localtime())[5,4];
    $year = $now_year;
    if ( $now_month != $month ) {
        $year--;
    }

    return timelocal($sec,$min,$hour,$day,$month,$year) + $time_shift * 60 * 60;
}

sub ut2str ($) {
    my @tmp_time=(localtime $_[0])[5,4,3,2,1,0];
    $tmp_time[0]+=1900;
    $tmp_time[1]+=1;
    #if ( $_[1] ) { $tmp_time[3] += $_[1] }
    return sprintf("%.4d/%.2d/%.2d %.2d:%.2d:%.2d", @tmp_time);
}

# - delta
# - err
# - warn
sub check_val($$$) {
    my ( $val, $err, $warn) = @_;
    my $color = 'green';

    if ( $val > $err ) {
        $color = 'red';
    } elsif ( $val > $warn ) {
        $color = 'yellow';
    }

    return $color;
}

# - list of colors
sub worst_color(@) {
    my $color='green';
    while (my $tmp_color = shift) {
        if ( $tmp_color eq 'red' ) {
            $color = 'red'
        } elsif ( ( $tmp_color eq 'yellow' ) and ( $color ne 'red' ) ) {
            $color = 'yellow'
        }
    }
    return $color;
}

# - hostname
sub init_agent_data($) {
    my $agent = shift;
    if ( not exists $data->{'agents'}->{$agent} ){
        $data->{'agents'}->{$agent}  = {
            'last_manifest_require' => 0,
            'last_compile_time' => 0,
            'last_apply_time'   => 0,
        }
    }
}

sub init_data() {
    print "WARN: initialize data\n";
    $data = {
        'pos'   => '0',
        'agents'    => {
        }
    };
}

sub send_xymon($){
    my $report = $_[0];
    my $xymon = $ENV{'XYMON'} ? $ENV{'XYMON'} : '/usr/lib/xymon/server/bin/xymon';
    my $xymsrv = $ENV{'XYMSRV'} ? $ENV{'XYMSRV'} : '10.1.200.35';
    system "$xymon $xymsrv \"$report\""
}

die "must be run from xymon" if not $ENV{'XYMON'};

# parse cli options
my $agent_to_delete;
Getopt::Long::Configure ('bundling');
GetOptions (
    'd=s'   => \$agent_to_delete,
);

# load data
if ( not -f $store_file ) {
    # file does not exist, initialize $data
    init_data();
    store($data, $store_file);
}
$data = retrieve($store_file);
init_data unless $data; # load was not successfull, initialize data

# delete node data, if required
if ( $agent_to_delete ) {
    delete $data->{'agents'}->{$agent_to_delete};
}

# parse log
open my $LOG, $puppet_log_file;
go_last_pos($LOG, $data->{'pos'});

while ($_=<$LOG>) {
    if ( m/Compiled catalog for (\S+) .* in ([0-9\.]+) seconds$/  ) {
        my $agent = $1;
        init_agent_data($agent);
        $data->{'agents'}->{$agent}->{'last_compile_time'} = $2;
        $data->{'agents'}->{$agent}->{'last_manifest_require'} = str2ut($_);
    } elsif ( m/\(\/\/(\S+?)\/Puppet\) Finished catalog run in ([0-9\.]+) seconds$/ ) {
        my $agent = $1;
        init_agent_data($agent);
        $data->{'agents'}->{$agent}->{'last_apply_time'} = $2;
    }
}

# save position
$data->{'pos'} = tell($LOG);
close $LOG;

#print (Dumper($data), "\n");

# analyze data and print result;
my $now = time();
my @colors;
my $message;

$message .= "<table border=1 cellpadding=5>\n<tr><td>agent</td><td>last catalog request</td><td>catalog compile time</td><td>manifest apply time</td></tr>\n";
foreach my $agent ( sort  keys %{ $data->{'agents'} } ) {
    my ($color_req, $color_cmp, $color_apl) = ('green', 'green', 'green');
    my $val;

    $val = $now - $data->{'agents'}->{$agent}->{'last_manifest_require'};
    $color_req = check_val($val, $err_last_require, $warn_last_require);

    $val = $data->{'agents'}->{$agent}->{'last_compile_time'};
    $color_cmp = check_val($val, $err_compile, $warn_compile);

    $val = $data->{'agents'}->{$agent}->{'last_apply_time'};
    $color_apl = check_val($val, $err_apply, $warn_apply);

    my $color = worst_color($color_req, $color_cmp, $color_apl);
    push @colors, $color;

    $message .= sprintf ("<tr><td>%s</td>", $agent);
    $message .= sprintf ("<td>&%s %10d sec ago (%-20s)</td>",
        $color_req,
        $now - $data->{'agents'}->{$agent}->{'last_manifest_require'} ,
        ut2str ( $data->{'agents'}->{$agent}->{'last_manifest_require'} )
    );
    $message .= sprintf ( "<td>&%s %s</td>",
        $color_cmp, $data->{'agents'}->{$agent}->{'last_compile_time'}
    );
    $message .= sprintf ("<td>&%s %s</td></tr>\n",
        $color_apl, $data->{'agents'}->{$agent}->{'last_apply_time'}
    );

}
$message .= "</table>\n";
$message .= "check_puppet_log.pl, v$version";

#send_xymon sprintf ( "status %s.%s %s %s\n%s", $host, $test, worst_color(@colors), ut2str($now), $message  );
send_xymon sprintf ( "status %s.%s %s %s\n%s", $host, $test, worst_color(@colors), ut2str($now), $message  );

# save data
store($data, $store_file);