daily pastebin goal
13%
SHARE
TWEET

memcron.pl 0.5.5

a guest Mar 12th, 2010 632 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/env perl
  2.  
  3. #////////////////////////////////////////////////////////////////////////////////////
  4. #
  5. #   memCron - DreamHost PS Memory Manager
  6. #
  7. #   Author     : Yaosan Yeo
  8. #   Updated by : Ben
  9. #   Version    : 0.5.5
  10. #   Date       : 12 March 2010
  11. #   URL        : http://memcron.com/
  12. #
  13. #   History  :
  14. #   0.5.5 [2010.03.12] - Simplified memory collecting for now
  15. #   0.5.4 [2010.03.11] - Update for DH changes.  Removes burst, combines base and swap to represent the number on the slider.
  16. #   0.5.3 [2010.02.09] - Get memory sizes from /proc/PID/statm
  17. #   0.5.2 [2010.02.04] - Removed depenency on Mail::SendMail
  18. #   0.5.1 [2010.02.03] - Added: Sanity checks on /proc/meminfo, calculate mem_size and mem_swap based on total rather than reported values
  19. #   0.5 [2010.01.22]   - Replaced most external calls with builtins
  20. #
  21. #   0.4.1 [2009.08.10 ] - Fixed: Memory calculation due to the 2x increase of swap memory by DreamHost
  22. #
  23. #   0.4 [2009.06.28] - Added: New parameter "$max_memory" and "$min_memory" to limit the memory range that memCron should operate within
  24. #                    - Added: New parameter "$mem_threshold" to set a memory threshold, which when exceeded, will send out notification to $email
  25. #                    - Added: 4 new subroutines - round(), get_mem_size(), get_mem_size(), get_mem_target() to simplify the code structure
  26. #                    - Added: Base and swap memory size to function output
  27. #                    - Added: Resize information to function output
  28. #                    - Changed: "$debug" parameter moved from config.cfg to this file
  29. #                    - Removed: "$username" parameter, now DreamHost API only needs API key for authentication
  30. #                    - Fixed: Memory target are now computed more correctly during downsize
  31. #
  32. #   0.3 [2009.05.27] - Added: New parameter "$downsize_resistance" to prevent rapid fluctuations from causing multiple resizes within short period of time
  33. #                    - Changed: 0.3 to 0.7 ratio between "$mem_mean" and "$mem_used instead" of 0.5 to 0.5 when calculating "$mem_free_tolerance"
  34. #                    - Changed: More descriptive error message when log file cannot be opened
  35. #                    - Fixed: "$unix_time" and "$cpu_load" will now be reevaluated until it is defined to ensure data point is always logged (removed in 0.5)
  36. #                    - Fixed: Memory target calculations now correctly accounts max and swap memory, which is capped at 4000 MB and 450MB respectively
  37. #
  38. #   0.2 [2009.05.03] - Fixed: Account for "$num_data" in z-score calculation to ensure number of memory changes is minimized to within daily limits
  39. #                    - Changed: "$use_api" is changed to "$debug", which if set disables all log outputs and API calls
  40. #
  41. #   0.1 [2009.04.30] - Initial release
  42. #
  43. #////////////////////////////////////////////////////////////////////////////////////
  44.  
  45. use strict;
  46. use warnings;
  47. no warnings 'uninitialized';
  48.  
  49. use LWP 5.64;
  50. use File::Basename;
  51. use Time::Local;
  52.  
  53. #////////////////////////////////////////////////// get configurations
  54.  
  55. my $debug = $ARGV[0] eq '-d';
  56.  
  57. # settings from config file
  58.  
  59. our (
  60.     $key,                 $ps,                  $cron_interval,
  61.     $max_memory,          $min_memory,          $mem_threshold,
  62.     $email,               $mem_free_confidence, $mem_free_to_used_ratio,
  63.     $downsize_resistance, $num_data,            $num_data_csv
  64. );
  65.  
  66. my $SAMPLES = $debug ? 1 : 60;
  67.  
  68. my $dir = dirname($0);
  69.  
  70. do "$dir/config.cfg";
  71.  
  72. # include z-score function
  73. require "$dir/zscore.pl";
  74.  
  75. # Log directory name
  76. my $log_dir = "$dir/logs";
  77.  
  78. # DreamHost PS limit
  79. my $daily_change_limit = 30;
  80. my $runs_per_day       = 1440 / $cron_interval;
  81.  
  82. # DreamHost PS defaults
  83. if ( $max_memory > 4000 ) { $max_memory = 4000; }
  84. if ( $min_memory < 300 )  { $min_memory = 300; }
  85.  
  86. # Statistics values
  87. my $mem_used_zscore = abs(
  88.     normsinv(
  89.         ( $daily_change_limit * $num_data / $runs_per_day ) / $runs_per_day
  90.     )
  91. );    #1.89;
  92. my $mem_free_zscore = abs( normsinv($mem_free_confidence) );    #3.09;
  93.  
  94. #////////////////////////////////////////////////// main program starts
  95.  
  96. sub cpu_load {
  97.     open( LOAD, "/proc/loadavg" ) || die "Unable to open /proc/loadavg";
  98.     my @l = split /\s+/, <LOAD>;
  99.     close(LOAD);
  100.     return @l;
  101. }
  102.  
  103. sub meminfo {
  104.     while (1) {
  105.         open( MEM, "/proc/meminfo" ) || die "Unable to open /proc/meminfo";
  106.         my %mem = map { (/(\w+):\s*(\d+)/) } <MEM>;
  107.         close(MEM);
  108.         ## kB to MB
  109.         for ( values %mem ) { $_ = int( $_ / 1024 ) }
  110.         ## sanity check MemTotal and retry if it's not valid
  111.         return \%mem if $mem{MemTotal} >= 150 && $mem{MemTotal} <= 4000;
  112.     }
  113. }
  114.  
  115. my $unix_time = time();
  116.  
  117. my $log_memory = "$log_dir/$ps.log";
  118.  
  119. my ( $mem_total, $mem_used, $mem_free, $cpu_load );
  120.  
  121. my $mem = meminfo();
  122. $mem_total = $mem->{MemTotal} + $mem->{SwapTotal};
  123.  
  124. foreach ( 1 .. $SAMPLES ) {
  125.     $cpu_load += ( cpu_load() )[1];
  126.     $mem = meminfo();
  127.     $mem_used += $mem_total - $mem->{MemFree} - $mem->{SwapFree} - $mem->{Cached};
  128.     sleep 1;
  129. }
  130.  
  131. $mem_used = round( $mem_used / $SAMPLES );
  132. $cpu_load = round( $cpu_load / $SAMPLES, 2 );
  133. $mem_free = $mem_total - $mem_used;
  134.  
  135. # write to memory log file
  136. unless ($debug) {
  137.     open( LOG, ">>$log_memory" ) || die "Cannot open memory log file!";
  138.     print LOG "$mem_used\n";
  139.     close(LOG);
  140. }
  141.  
  142. #////////////// truncate memory log
  143.  
  144. system("tail -n $num_data $log_memory > $log_memory.temp");
  145. rename "$log_memory.temp", $log_memory;
  146.  
  147. #////////////// calculate the amount of memory to be reserved based on past data
  148.  
  149. open( LOG, "$log_memory" ) || die "Cannot open memory log file!";
  150. my @data = <LOG>;
  151. close(LOG);
  152.  
  153. # construct a short list to be used when downsizing
  154. my @shortlist =
  155.   @data[ ( scalar(@data) - $downsize_resistance ) .. ( scalar(@data) - 1 ) ];
  156.  
  157. # make sure we have enough data points before proceeding
  158. if ( scalar(@data) < round( $num_data * 0.1 ) ) { exit 0; }
  159.  
  160. my ( $mem_stattotal, $mem_mean, $mem_median, $mem_stdev ) = stats(@data);
  161.  
  162. $mem_mean = round($mem_mean);
  163. $mem_stdev = round( $mem_stdev, 2 );
  164.  
  165. my $mem_used_tolerance = round( $mem_used_zscore * $mem_stdev );
  166. my $mem_free_tolerance = round( $mem_free_zscore * $mem_stdev );
  167.  
  168. unless ( $mem_free_tolerance > ( $mem_mean * 0.3 + $mem_used * 0.7 ) * $mem_free_to_used_ratio )
  169. {
  170.     $mem_free_tolerance = round( ( $mem_mean * 0.3 + $mem_used * 0.7 ) * $mem_free_to_used_ratio );
  171. }
  172.  
  173. #////////////// calculate the memory required
  174.  
  175. my $mem_required = $mem_used + $mem_free_tolerance;
  176. my $mem_target   = get_mem_target($mem_required);
  177. my $set = ( abs( $mem_total - $mem_required ) > $mem_used_tolerance ) ? 1 : 0;
  178.  
  179. #////////////// downsize resistance
  180.  
  181. my $downsize_count = 0;
  182.  
  183. my $log_downsize = "$log_dir/downsize.log";
  184.  
  185. if ( ( $mem_target < $mem_total ) && $set ) {
  186.     if ( -e "$log_downsize" ) {
  187.         open( LOG, "$log_downsize" ) || die "Cannot open downsize log file!";
  188.         my @data = <LOG>;
  189.         close(LOG);
  190.         $downsize_count = shift(@data);
  191.         $downsize_count++;
  192.     }
  193.     else {
  194.         $downsize_count = 1;
  195.     }
  196.  
  197.     if ( $downsize_count >= $downsize_resistance ) {
  198.         $downsize_count = 0;
  199.  
  200.         $mem_required = round( ( stats(@shortlist) )[1] ) + $mem_free_tolerance;
  201.         $mem_target = get_mem_target($mem_required);
  202.     }
  203.     else {
  204.         $set = 0;
  205.     }
  206. }
  207.  
  208. unless ($debug) {
  209.     open( LOG, ">$log_downsize" ) || die "Cannot open downsize log file!";
  210.     print LOG "$downsize_count";
  211.     close(LOG);
  212. }
  213.  
  214. #////////////// set memory using DreamHost API, output result
  215.  
  216. my $sendmail;
  217. my $date    = date_time_string();
  218. my $message = <<"MESSAGE";
  219. Total: $mem_total       Used: $mem_used Free: $mem_free
  220. Mean: $mem_mean Stdev: $mem_stdev       Load: $cpu_load
  221. Tolerance: $mem_used_tolerance  Tolerance (free): $mem_free_tolerance
  222.  
  223. Target: $mem_target
  224. MESSAGE
  225.  
  226. $set = 0 if $mem_target == $mem_total || $debug;
  227.  
  228. if ($set) {
  229.  
  230.     $message .= "\nMemory was resized from $mem_total MB to $mem_target MB!\n";
  231.  
  232.     # check against memory threshold, send notification email if it is exceeded
  233.     if ( $mem_threshold && $mem_target > $mem_threshold ) {
  234.         print "\n*** Warning! Threshold exceeded! ***\n";
  235.         $sendmail = 1 unless $debug;
  236.     }
  237.  
  238.     # use API to resize memory
  239.     my $ua       = LWP::UserAgent->new;
  240.     my $url      = 'https://panel.dreamhost.com/api/';
  241.     my $response = $ua->post(
  242.         $url,
  243.         {
  244.             key       => $key,
  245.             cmd       => 'dreamhost_ps-set_size',
  246.             unique_id => $unix_time . $$,
  247.             ps        => $ps,
  248.             size      => $mem_target
  249.         }
  250.     );
  251.     $mem_total = get_mem_total($mem_target);
  252.  
  253.     $response->is_success || die "Error at $url\n ", $response->status_line,
  254.       "\n Aborting";
  255.     $message .= "API Response: \n" . $response->content . "\n";
  256.  
  257.     # send notification if necessary
  258.     if ( $sendmail && $email ) {
  259.         my $top = `top -b -n 1`;
  260.         chomp($top);
  261.  
  262.         open( MAIL, "/usr/bin/sendmail -t" )
  263.           or die "Unable to open sendmail: $!";
  264.         print MAIL <<"MESSAGE";
  265. To: $email
  266. From: memCron <noreply\@memcron.com>
  267. Date: $date
  268. Subject: Memory threshold exceeded for $ps
  269.  
  270. $message
  271.  
  272. -----
  273.  
  274. List of running processes:
  275.  
  276. $top
  277.  
  278. This report is generated by memCron at $date.
  279. MESSAGE
  280.  
  281.         close(MAIL);
  282.         $message .= "\nNotification sent to $email.\n";
  283.     }
  284.  
  285. }
  286. else {
  287.  
  288.     $message .= "\nNo need to adjust memory"
  289.       . (
  290.         ($downsize_count)
  291.         ? " yet! ($downsize_count/$downsize_resistance)"
  292.         : "!"
  293.       ) . "\n";
  294. }
  295.  
  296. print $message;
  297. print "\n-----\n";
  298.  
  299. #////////////// write memory usage and cpu load to csv file
  300.  
  301. my $log_csv = "$log_dir/$ps.csv";
  302.  
  303. my $log_line = join( ",",
  304.     $unix_time,          $mem_total,          $mem_used,
  305.     $cpu_load,           $mem_mean,           $mem_stdev,
  306.     $mem_used_tolerance, $mem_free_tolerance, $downsize_count );
  307. if ($debug) {
  308.     print "$log_line\n";
  309. }
  310. else {
  311.     open( LOG, ">>$log_csv" ) || die "Cannot open csv file!";
  312.     print LOG $log_line, "\n";
  313.     close(LOG);
  314. }
  315.  
  316. #////////////// truncate csv log
  317.  
  318. if ( -e $log_csv ) {
  319.     system("tail -n $num_data_csv $log_csv > $log_csv.temp");
  320.     system("mv $log_csv.temp $log_csv");
  321. }
  322.  
  323. #////////////// subroutines
  324.  
  325. sub round {
  326.     my ( $val, $places ) = @_;
  327.     $places ||= 0;
  328.     return sprintf( "%.${places}f", $val || 0 );
  329. }
  330.  
  331. sub get_mem_target {
  332.     my $mem_target = shift;
  333.  
  334.     # handle max and min memory limit
  335.     if ( $mem_target < $min_memory ) {
  336.         $mem_target = $min_memory;
  337.     }
  338.     elsif ( $mem_target > $max_memory ) {
  339.         $mem_target = $max_memory;
  340.     }
  341.  
  342.     return ( round($mem_target) );
  343. }
  344.  
  345. sub stats {
  346.     my $total = 0;
  347.     foreach my $v (@_) { $total += $v; }
  348.  
  349.     my $average = $total / @_;
  350.     my $median =
  351.         @_ % 2
  352.       ? $_[ ( @_ - 1 ) / 2 ]
  353.       : ( $_[ @_ / 2 - 1 ] + $_[ @_ / 2 ] ) / 2;
  354.  
  355.     my $sqtotal = 0;
  356.     foreach my $v (@_) {
  357.         $sqtotal += ( $average - $v )**2;
  358.     }
  359.     my $stdev = ( $sqtotal / @_ )**0.5;
  360.  
  361.     return ( $total, $average, $median, $stdev );
  362. }
  363.  
  364. sub date_time_string {
  365.  
  366.     # convert a time() value to a date-time string according to RFC 822
  367.  
  368.     my $time = $_[0] || time();    # default to now if no argument
  369.  
  370.     my @months = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
  371.     my @wdays  = qw(Sun Mon Tue Wed Thu Fri Sat);
  372.  
  373.     my ( $sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst ) =
  374.       localtime($time);
  375.  
  376.     my $TZ = $ENV{TZ};
  377.     if ( $TZ eq "" ) {
  378.  
  379.         # offset in hours
  380.         my $offset = sprintf "%.1f", ( timegm(localtime) - time ) / 3600;
  381.         my $minutes = sprintf "%02d", abs( $offset - int($offset) ) * 60;
  382.         $TZ = sprintf( "%+03d", int($offset) ) . $minutes;
  383.     }
  384.     return join( " ",
  385.         ( $wdays[$wday] . ',' ),
  386.         $mday, $months[$mon],
  387.         $year + 1900,
  388.         sprintf( "%02d:%02d:%02d", $hour, $min, $sec ), $TZ );
  389. }
RAW Paste Data
Top