SHOW:
|
|
- or go back to the newest paste.
| 1 | - | Скрипт для xymon. Мониторит состояние агентов puppet. Запускаться должен на хосте с puppet master'ом. |
| 1 | + | |
| 2 | - | http://www.linux.org.ru/forum/admin/8588141 |
| 2 | + | |
| 3 | # author: Oleg Korchagin | |
| 4 | # license: GPL v2 or newer | |
| 5 | # Скрипт для xymon. Мониторит состояние агентов puppet. | |
| 6 | # Запускаться должен на хосте с puppet master'ом. | |
| 7 | # http://www.linux.org.ru/forum/admin/8588141 | |
| 8 | ||
| 9 | ||
| 10 | ||
| 11 | use strict; | |
| 12 | use Storable; | |
| 13 | use Data::Dumper; | |
| 14 | use Time::Local; | |
| 15 | use Getopt::Std; | |
| 16 | use Getopt::Long; | |
| 17 | ||
| 18 | my $store_file='/usr/lib/xymon/client/data/check_puppet_log.store'; | |
| 19 | my $puppet_log_file='/var/log/messages'; | |
| 20 | ||
| 21 | # $data - data to load from and safe to $store_file | |
| 22 | # structure: | |
| 23 | # {
| |
| 24 | # pos - last position in $puppet_log_file | |
| 25 | # agents => {
| |
| 26 | # $agent => {
| |
| 27 | # last_manifest_require - last request of manifest, in unixtime | |
| 28 | # last_compile_time - last catalog coptile time for this node | |
| 29 | # last_apply_time - last manifest apply time on this node | |
| 30 | # } | |
| 31 | # } | |
| 32 | # } | |
| 33 | my $data; | |
| 34 | ||
| 35 | my $warn_last_require = 3600; | |
| 36 | my $err_last_require = 5400; | |
| 37 | ||
| 38 | my $warn_compile = 10; | |
| 39 | my $err_compile = 60; | |
| 40 | ||
| 41 | my $warn_apply = 200; | |
| 42 | my $err_apply = 600; | |
| 43 | ||
| 44 | my $host = $ENV{'MACHINE'} ? $ENV{'MACHINE'} : 'vml-puppet';
| |
| 45 | my $test = "puppet_agents"; | |
| 46 | ||
| 47 | my $version = '0.1a'; | |
| 48 | ||
| 49 | # - file | |
| 50 | # - position | |
| 51 | sub go_last_pos($$) {
| |
| 52 | my ( $FILE, $pos ) = @_; | |
| 53 | ||
| 54 | # check size | |
| 55 | seek( $FILE, 0, 2); # go to end of file | |
| 56 | ||
| 57 | seek( $FILE, tell($FILE) < $pos ? 0 : $pos, 0); # go to last position, if no logrotate. Or to start of file | |
| 58 | } | |
| 59 | ||
| 60 | # convert time to unixtime. string example: | |
| 61 | # Dec 9 03:33:01 vml-puppet-test rsyslogd: | |
| 62 | sub str2ut($) {
| |
| 63 | my $time_shift = 0; | |
| 64 | my %m=( | |
| 65 | "Jan" => 0, "Feb"=>1,"Mar"=>2,"Apr"=>3,"May"=>4,"Jun"=>5,"Jul"=>6, "Aug"=>7,"Sep"=>8,"Oct"=>9,"Nov"=>10,"Dec"=>11, | |
| 66 | "Янв" => 0, "Фев" => 1, "Мар" => 2, "Апр" => 3, "Май" => 4, "Июн" => 5, "Июл" => 6, "Авг" => 7, "Сен" => 8, "Окт" => 9, "Ноя" => 10, "Дек" => 11 | |
| 67 | ); | |
| 68 | $_[0] =~ /^(\S{3,6})\s+([0-9]{1,2})\s+([0-9]{2}):([0-9]{2}):([0-9]{2})\s/ || print "can't decode date$_[0]";
| |
| 69 | my ( $year, $month, $day, $hour, $min, $sec ) = (0,$m{$1},$2,$3,$4,$5);
| |
| 70 | ||
| 71 | my ( $now_year, $now_month ) = ( localtime())[5,4]; | |
| 72 | $year = $now_year; | |
| 73 | if ( $now_month != $month ) {
| |
| 74 | $year--; | |
| 75 | } | |
| 76 | ||
| 77 | return timelocal($sec,$min,$hour,$day,$month,$year) + $time_shift * 60 * 60; | |
| 78 | } | |
| 79 | ||
| 80 | sub ut2str ($) {
| |
| 81 | my @tmp_time=(localtime $_[0])[5,4,3,2,1,0]; | |
| 82 | $tmp_time[0]+=1900; | |
| 83 | $tmp_time[1]+=1; | |
| 84 | #if ( $_[1] ) { $tmp_time[3] += $_[1] }
| |
| 85 | return sprintf("%.4d/%.2d/%.2d %.2d:%.2d:%.2d", @tmp_time);
| |
| 86 | } | |
| 87 | ||
| 88 | # - delta | |
| 89 | # - err | |
| 90 | # - warn | |
| 91 | sub check_val($$$) {
| |
| 92 | my ( $val, $err, $warn) = @_; | |
| 93 | my $color = 'green'; | |
| 94 | ||
| 95 | if ( $val > $err ) {
| |
| 96 | $color = 'red'; | |
| 97 | } elsif ( $val > $warn ) {
| |
| 98 | $color = 'yellow'; | |
| 99 | } | |
| 100 | ||
| 101 | return $color; | |
| 102 | } | |
| 103 | ||
| 104 | # - list of colors | |
| 105 | sub worst_color(@) {
| |
| 106 | my $color='green'; | |
| 107 | while (my $tmp_color = shift) {
| |
| 108 | if ( $tmp_color eq 'red' ) {
| |
| 109 | $color = 'red' | |
| 110 | } elsif ( ( $tmp_color eq 'yellow' ) and ( $color ne 'red' ) ) {
| |
| 111 | $color = 'yellow' | |
| 112 | } | |
| 113 | } | |
| 114 | return $color; | |
| 115 | } | |
| 116 | ||
| 117 | # - hostname | |
| 118 | sub init_agent_data($) {
| |
| 119 | my $agent = shift; | |
| 120 | if ( not exists $data->{'agents'}->{$agent} ){
| |
| 121 | $data->{'agents'}->{$agent} = {
| |
| 122 | 'last_manifest_require' => 0, | |
| 123 | 'last_compile_time' => 0, | |
| 124 | 'last_apply_time' => 0, | |
| 125 | } | |
| 126 | } | |
| 127 | } | |
| 128 | ||
| 129 | sub init_data() {
| |
| 130 | print "WARN: initialize data\n"; | |
| 131 | $data = {
| |
| 132 | 'pos' => '0', | |
| 133 | 'agents' => {
| |
| 134 | } | |
| 135 | }; | |
| 136 | } | |
| 137 | ||
| 138 | sub send_xymon($){
| |
| 139 | my $report = $_[0]; | |
| 140 | my $xymon = $ENV{'XYMON'} ? $ENV{'XYMON'} : '/usr/lib/xymon/server/bin/xymon';
| |
| 141 | my $xymsrv = $ENV{'XYMSRV'} ? $ENV{'XYMSRV'} : '10.1.200.35';
| |
| 142 | system "$xymon $xymsrv \"$report\"" | |
| 143 | } | |
| 144 | ||
| 145 | die "must be run from xymon" if not $ENV{'XYMON'};
| |
| 146 | ||
| 147 | # parse cli options | |
| 148 | my $agent_to_delete; | |
| 149 | Getopt::Long::Configure ('bundling');
| |
| 150 | GetOptions ( | |
| 151 | 'd=s' => \$agent_to_delete, | |
| 152 | ); | |
| 153 | ||
| 154 | # load data | |
| 155 | if ( not -f $store_file ) {
| |
| 156 | # file does not exist, initialize $data | |
| 157 | init_data(); | |
| 158 | store($data, $store_file); | |
| 159 | } | |
| 160 | $data = retrieve($store_file); | |
| 161 | init_data unless $data; # load was not successfull, initialize data | |
| 162 | ||
| 163 | # delete node data, if required | |
| 164 | if ( $agent_to_delete ) {
| |
| 165 | delete $data->{'agents'}->{$agent_to_delete};
| |
| 166 | } | |
| 167 | ||
| 168 | # parse log | |
| 169 | open my $LOG, $puppet_log_file; | |
| 170 | go_last_pos($LOG, $data->{'pos'});
| |
| 171 | ||
| 172 | while ($_=<$LOG>) {
| |
| 173 | if ( m/Compiled catalog for (\S+) .* in ([0-9\.]+) seconds$/ ) {
| |
| 174 | my $agent = $1; | |
| 175 | init_agent_data($agent); | |
| 176 | $data->{'agents'}->{$agent}->{'last_compile_time'} = $2;
| |
| 177 | $data->{'agents'}->{$agent}->{'last_manifest_require'} = str2ut($_);
| |
| 178 | } elsif ( m/\(\/\/(\S+?)\/Puppet\) Finished catalog run in ([0-9\.]+) seconds$/ ) {
| |
| 179 | my $agent = $1; | |
| 180 | init_agent_data($agent); | |
| 181 | $data->{'agents'}->{$agent}->{'last_apply_time'} = $2;
| |
| 182 | } | |
| 183 | } | |
| 184 | ||
| 185 | # save position | |
| 186 | $data->{'pos'} = tell($LOG);
| |
| 187 | close $LOG; | |
| 188 | ||
| 189 | #print (Dumper($data), "\n"); | |
| 190 | ||
| 191 | # analyze data and print result; | |
| 192 | my $now = time(); | |
| 193 | my @colors; | |
| 194 | my $message; | |
| 195 | ||
| 196 | $message .= "<table border=1 cellpadding=5>\n<tr><td>agent</td><td>last catalog request</td><td>catalog compile time</td><td>manifest apply time</td></tr>\n"; | |
| 197 | foreach my $agent ( sort keys %{ $data->{'agents'} } ) {
| |
| 198 | my ($color_req, $color_cmp, $color_apl) = ('green', 'green', 'green');
| |
| 199 | my $val; | |
| 200 | ||
| 201 | $val = $now - $data->{'agents'}->{$agent}->{'last_manifest_require'};
| |
| 202 | $color_req = check_val($val, $err_last_require, $warn_last_require); | |
| 203 | ||
| 204 | $val = $data->{'agents'}->{$agent}->{'last_compile_time'};
| |
| 205 | $color_cmp = check_val($val, $err_compile, $warn_compile); | |
| 206 | ||
| 207 | $val = $data->{'agents'}->{$agent}->{'last_apply_time'};
| |
| 208 | $color_apl = check_val($val, $err_apply, $warn_apply); | |
| 209 | ||
| 210 | my $color = worst_color($color_req, $color_cmp, $color_apl); | |
| 211 | push @colors, $color; | |
| 212 | ||
| 213 | $message .= sprintf ("<tr><td>%s</td>", $agent);
| |
| 214 | $message .= sprintf ("<td>&%s %10d sec ago (%-20s)</td>",
| |
| 215 | $color_req, | |
| 216 | $now - $data->{'agents'}->{$agent}->{'last_manifest_require'} ,
| |
| 217 | ut2str ( $data->{'agents'}->{$agent}->{'last_manifest_require'} )
| |
| 218 | ); | |
| 219 | $message .= sprintf ( "<td>&%s %s</td>", | |
| 220 | $color_cmp, $data->{'agents'}->{$agent}->{'last_compile_time'}
| |
| 221 | ); | |
| 222 | $message .= sprintf ("<td>&%s %s</td></tr>\n",
| |
| 223 | $color_apl, $data->{'agents'}->{$agent}->{'last_apply_time'}
| |
| 224 | ); | |
| 225 | ||
| 226 | } | |
| 227 | $message .= "</table>\n"; | |
| 228 | $message .= "check_puppet_log.pl, v$version"; | |
| 229 | ||
| 230 | #send_xymon sprintf ( "status %s.%s %s %s\n%s", $host, $test, worst_color(@colors), ut2str($now), $message ); | |
| 231 | send_xymon sprintf ( "status %s.%s %s %s\n%s", $host, $test, worst_color(@colors), ut2str($now), $message ); | |
| 232 | ||
| 233 | # save data | |
| 234 | store($data, $store_file); |