Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python3
- # this script acts as watchdog for a specified systemd service
- # tested on debian 9 - may work on other linux distros
- # this script runs as a systemd service
- # configuration of the service is done through the systemd unit file
- # email notifications require local smtp services
- # Run the following commands to setup and parameterise the watchdog service
- """
- # change this path to where you will store and run this script
- watchdogpath=/opt/watchdog.py
- # change this variable to specify which service you want the watchdog to check
- servicename=apache2
- # the following section sets up the systemd unit file for the watchdog service
- cat > /etc/systemd/system/watchdog.service << EOF
- [Unit]
- Description=Watchdog
- After=${servicename}.service
- [Service]
- Type=simple
- # change these environment variables to suit your requirements
- Environment=SERVICENAME=${servicename}
- Environment=CHECKINTERVAL=60
- Environment=RESTARTATTEMPTINTERVAL=15
- Environment=RESTARTATTEMPTRETRIES=4
- Environment=NOTIFICATIONFROMEMAIL=mail@mydomain.com
- Environment=NOTIFICATIONTOEMAIL=mail@mydomain.com
- ExecStart=${watchdogpath}
- Restart=always
- [Install]
- WantedBy=multi-user.target
- EOF
- # run these commands to enable the service
- chmod 755 ${watchdogpath}
- systemctl enable watchdog.service
- # note: this command needs to be run any time the watchdog.service systemd script changes
- systemctl daemon-reload
- # start and check the service with
- systemctl restart watchdog.service
- journalctl --follow -u watchdog.service
- """
- # Running the following commands will enable you to
- # run/test the script directly from the shell
- """
- # These Variables can be set in the shell if you want
- # to test the script directly from the shell
- export SERVICENAME=apache2
- export CHECKINTERVAL=60
- export RESTARTATTEMPTINTERVAL=15
- export RESTARTATTEMPTRETRIES=4
- export NOTIFICATIONFROMEMAIL=postmaster@fqdn.net
- export NOTIFICATIONTOEMAIL=postmaster@fqdn.net
- # Otherwise, the variables should be set in the systemd
- """
- import os
- import sys
- import subprocess
- import time
- import logging
- import re
- import smtplib
- from email.mime.text import MIMEText
- emailactive=False # set this to true if if you have email configured locally
- logging.basicConfig(level="INFO")
- def checkposint(val):
- try:
- val=int(val)
- val > 0
- except:
- logging.warning("Config Error: Interval and Attempt variables must be positive integers")
- return val
- servicename = os.environ.get("SERVICENAME", None)
- checkinterval = checkposint(os.environ.get("CHECKINTERVAL", None))
- restartinterval = checkposint(os.environ.get("RESTARTATTEMPTINTERVAL", None))
- restartattemptretries = checkposint(os.environ.get("RESTARTATTEMPTRETRIES", None))
- notificationfromemail = os.environ.get("NOTIFICATIONFROMEMAIL", None)
- notificationtoemail = os.environ.get("NOTIFICATIONTOEMAIL", None)
- # check if servicename appears valid
- with open(os.devnull, 'wb') as hide_output:
- var = subprocess.check_output(['systemctl', 'list-units'])
- if not (servicename + ".service" in str(var)) :
- logging.warning("Servicename does not appear to be valid. Please check...")
- sys.exit(1)
- # check the timing variables
- if None in (servicename, checkinterval, restartinterval, restartattemptretries, notificationfromemail, notificationtoemail):
- logging.warning("One or more configuration variables missing. Please check...")
- sys.exit(1)
- # check that the email is valid
- for tempemail in [notificationtoemail, notificationfromemail ]:
- if not re.match("[^@]+@[^@]+\.[^@]+", tempemail):
- logging.warning("An Email configured for notifications is invalid. Please check...")
- sys.exit(1)
- def emailthis(message,fromemail,toemail,servicename):
- servicealert = 'ServiceAlert : ' + servicename + " on " + os.uname()[1]
- msg['Subject'] = servicealert % message
- msg['From'] = fromemail
- msg['To'] = toemail
- # Send the message via our own SMTP server, but don't include the envelope header.
- s = smtplib.SMTP('localhost')
- s.sendmail(fromemail, [toemail], msg.as_string())
- s.quit()
- def service_command(servicename,action):
- with open(os.devnull, 'wb') as hide_output:
- return subprocess.Popen(['service', servicename, action], stdout=hide_output, stderr=hide_output).wait()
- def service_running(servicename):
- exit_code = service_command(servicename,'status')
- return exit_code == 0
- def restart_service(servicename):
- exit_code = service_command(servicename,'restart')
- return exit_code == 0
- logging.info('Watchdog service name is ' + servicename)
- logging.info('Watchdog check interval is ' + str(checkinterval))
- logging.info('Watchdog restart interval is ' + str(restartinterval))
- logging.info('Watchdog restart attempt retries is ' + str(restartattemptretries))
- logging.info('Watchdog notification from email is ' + notificationfromemail)
- logging.info('Watchdog notification to email is ' + notificationtoemail)
- if service_running(servicename):
- message = "Service " + servicename + " is running"
- logging.info(message)
- while True:
- while service_running(servicename):
- time.sleep(checkinterval)
- message = "Service " + servicename + " is down"
- logging.warning(message)
- if emailactive: emailthis(message)
- restartattemptcount=0
- while restartattemptcount < restartattemptretries:
- restartattemptcount=restartattemptcount+1
- message = "Attempting restart #" + str(restartattemptcount) + " of " + servicename
- logging.warning(message)
- restart_service(servicename)
- if service_running(servicename):
- message = "Service " + servicename + " is back up and running after " + str(restartattemptcount) + " attempts"
- logging.info(message)
- if emailactive: emailthis(message)
- break
- time.sleep(restartinterval)
- if not service_running(servicename):
- message = "Service " + servicename + " can\'t be started after " + str(restartattemptcount) + " attempts"
- if emailactive: emailthis(message)
- logging.warning(message)
- time.sleep(checkinterval)
Add Comment
Please, Sign In to add comment