Advertisement
Guest User

Untitled

a guest
Nov 21st, 2015
74
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.51 KB | None | 0 0
  1. #!/usr/bin/python
  2.  
  3. import subprocess
  4. import time
  5. import argparse
  6. import sys
  7. import tempfile
  8. import smtplib
  9. import time
  10. import re
  11. from email.mime.text import MIMEText
  12.  
  13. def getDiskUtil(disk):
  14. ''' The average time (in milliseconds) for I/O requests issued to the device to be served.'''
  15. disk = disk[:-1]
  16. iostat = subprocess.check_output(["iostat","-x",disk, "1", "3"])
  17. for line in iostat.split('\n'):
  18. if disk.split('/')[2] in line:
  19. await = line.split()[9]
  20. return float(await)
  21.  
  22. def sendAlert(mount, usage, await):
  23. tf = tempfile.NamedTemporaryFile(delete=True)
  24. with open(tf.name, "w") as fp:
  25. fp.write("Disk alert on volume {} usage {} await {}\n".format(mount, usage, await))
  26. fp.close()
  27.  
  28. with open(tf.name, "r") as fp:
  29. msg = MIMEText(fp.read())
  30. msg['Subject'] = 'Disk utilization alert'
  31. msg['From'] = "alert@testing.aws.orchardplatform.com"
  32. msg['To'] = "rodrick.brown@gmail.com"
  33. s = smtplib.SMTP('localhost')
  34. s.sendmail("alert@testing.aws.orchardplatform.com", ["rodrick.brown@gmail.com"], msg.as_string())
  35. s.quit()
  36.  
  37. def logWriter(logfile,mount,usage,await):
  38. timestamp = time.strftime("%d/%m/%y %H:%M:%S")
  39. with open(logfile.name, "a+") as fp:
  40. timestamp = time.strftime("%d/%m/%y %H:%M:%S")
  41. fp.write("Volume {} went critical usage: {} await: {} at {}\n".format(mount,usage,await,timestamp))
  42.  
  43. if __name__ == '__main__':
  44. parser = argparse.ArgumentParser(description='Process disk checks')
  45. parser.add_argument('--diskusage', action="store", dest="diskusage", type=float)
  46. parser.add_argument('--iorequest', action="store", dest="iorequest", type=float)
  47.  
  48. args = parser.parse_args()
  49. if not args.diskusage:
  50. sys.exit(-1)
  51.  
  52. logfile = tempfile.NamedTemporaryFile(delete=False)
  53. print("--- Writing alerts to {} ---".format(logfile.name))
  54. while True:
  55. df = subprocess.check_output(["df", "-m"])
  56. alert_triggers = set()
  57.  
  58. for line in df.split('\n'):
  59. if line.startswith('/dev'):
  60. device = line.split()[0]
  61. usage = line.split()[4]
  62. mount = line.split()[5]
  63. usage_value = float(usage[:-1])
  64.  
  65. await = getDiskUtil(device)
  66. if await > float(args.iorequest):
  67. print("io threshold triggered {} {}".format(await, device))
  68. logWriter(logfile,mount,usage,await)
  69. alert_triggers.add(mount)
  70. sendAlert(mount, usage, await)
  71.  
  72. if usage_value > float(args.diskusage):
  73. print("disk usage threshold triggered: {} {} {}".format(usage_value, args.diskusage, device))
  74. logWriter(logfile,mount,usage,await)
  75. alert_triggers.add(mount)
  76. sendAlert(mount, usage, await)
  77.  
  78. else:
  79. print("mount - {} usage: {} await: {} OK".format(mount, usage, await))
  80. if mount in alert_triggers:
  81. with open(logfile.name,"a+") as fp:
  82. fp.write("mount - {} usage: {} await: {} recovered".format(mount,usage,await))
  83. logWriter(logfile,mount,usage,await)
  84. time.sleep(3)
  85.  
  86. [rodrick@localhost ~]$ ./df.py --diskusage 20 --iorequest .1
  87. --- Writing alerts to /tmp/tmpXQpZi6 ---
  88. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  89. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  90. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  91. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  92. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  93. io threshold triggered 1.0 /dev/xvda1
  94. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  95. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
  96. disk usage threshold triggered: 43.0 20.0 /dev/xvda1
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement