#!/usr/bin/env python3
# this script acts as watchdog for a specified systemd service
# tested on debian 9 - may work on other linux distros
# this script runs as a systemd service
# configuration of the service is done through the systemd unit file
# email notifications require local smtp services
# Run the following commands to setup and parameterise the watchdog service

"""
# change this path to where you will store and run this script
watchdogpath=/opt/watchdog.py
# change this variable to specify which service you want the watchdog to check
servicename=apache2
# the following section sets up the systemd unit file for the watchdog service
cat > /etc/systemd/system/watchdog.service << EOF
[Unit]
Description=Watchdog
After=${servicename}.service
[Service]
Type=simple
# change these environment variables to suit your requirements
Environment=SERVICENAME=${servicename}
Environment=CHECKINTERVAL=60
Environment=RESTARTATTEMPTINTERVAL=15
Environment=RESTARTATTEMPTRETRIES=4
Environment=NOTIFICATIONFROMEMAIL=mail@mydomain.com
Environment=NOTIFICATIONTOEMAIL=mail@mydomain.com
ExecStart=${watchdogpath}
Restart=always
[Install]
WantedBy=multi-user.target
EOF
# run these commands to enable the service
chmod 755 ${watchdogpath}
systemctl enable watchdog.service
# note: this command needs to be run any time the watchdog.service systemd script changes
systemctl daemon-reload
# start and check the service with
systemctl restart watchdog.service
journalctl --follow -u watchdog.service
"""

# Running the following commands will enable you to 
# run/test the script directly from the shell
"""
# These Variables can be set in the shell if you want
# to test the script directly from the shell
export SERVICENAME=apache2
export CHECKINTERVAL=60
export RESTARTATTEMPTINTERVAL=15
export RESTARTATTEMPTRETRIES=4
export NOTIFICATIONFROMEMAIL=postmaster@fqdn.net
export NOTIFICATIONTOEMAIL=postmaster@fqdn.net
# Otherwise, the variables should be set in the systemd
"""

import os
import sys
import subprocess
import time
import logging
import re
import smtplib
from email.mime.text import MIMEText

emailactive=False  # set this to true if if you have email configured locally

logging.basicConfig(level="INFO")

def checkposint(val):
    try:
        val=int(val)
        val > 0
    except:
        logging.warning("Config Error: Interval and Attempt variables must be positive integers")
    return val

servicename = os.environ.get("SERVICENAME", None)
checkinterval = checkposint(os.environ.get("CHECKINTERVAL", None))
restartinterval = checkposint(os.environ.get("RESTARTATTEMPTINTERVAL", None))
restartattemptretries = checkposint(os.environ.get("RESTARTATTEMPTRETRIES", None))
notificationfromemail = os.environ.get("NOTIFICATIONFROMEMAIL", None)
notificationtoemail = os.environ.get("NOTIFICATIONTOEMAIL", None)

# check if servicename appears valid
with open(os.devnull, 'wb') as hide_output:
    var = subprocess.check_output(['systemctl', 'list-units'])
if not (servicename + ".service" in str(var)) :
    logging.warning("Servicename does not appear to be valid. Please check...")
    sys.exit(1)

# check the timing variables
if None in (servicename, checkinterval, restartinterval, restartattemptretries, notificationfromemail, notificationtoemail):
    logging.warning("One or more configuration variables missing. Please check...")
    sys.exit(1)

# check that the email is valid
for tempemail in [notificationtoemail, notificationfromemail ]:
    if not re.match("[^@]+@[^@]+\.[^@]+", tempemail):
        logging.warning("An Email configured for notifications is invalid. Please check...")
        sys.exit(1)

def emailthis(message,fromemail,toemail,servicename):
    servicealert = 'ServiceAlert : ' + servicename  + " on " + os.uname()[1]
    msg['Subject'] = servicealert % message
    msg['From'] = fromemail
    msg['To'] = toemail
    # Send the message via our own SMTP server, but don't include the envelope header.
    s = smtplib.SMTP('localhost')
    s.sendmail(fromemail, [toemail], msg.as_string())
    s.quit()

def service_command(servicename,action):
    with open(os.devnull, 'wb') as hide_output:
        return subprocess.Popen(['service', servicename, action], stdout=hide_output, stderr=hide_output).wait()

def service_running(servicename):
    exit_code = service_command(servicename,'status')
    return exit_code == 0

def restart_service(servicename):
    exit_code = service_command(servicename,'restart')
    return exit_code == 0

logging.info('Watchdog service name is ' + servicename)
logging.info('Watchdog check interval is ' + str(checkinterval))
logging.info('Watchdog restart interval is ' + str(restartinterval))
logging.info('Watchdog restart attempt retries is ' + str(restartattemptretries))
logging.info('Watchdog notification from email is ' + notificationfromemail)
logging.info('Watchdog notification to email is ' + notificationtoemail)

if service_running(servicename):
    message = "Service " + servicename + " is running"
    logging.info(message)
while True:
    while service_running(servicename):
        time.sleep(checkinterval)
    message = "Service " + servicename + " is down"
    logging.warning(message)
    if emailactive: emailthis(message)
    restartattemptcount=0
    while restartattemptcount < restartattemptretries:
        restartattemptcount=restartattemptcount+1
        message = "Attempting restart #" + str(restartattemptcount) + " of " + servicename
        logging.warning(message)
        restart_service(servicename)
        if service_running(servicename):
            message = "Service " + servicename + " is back up and running after " + str(restartattemptcount) + " attempts"
            logging.info(message)
            if emailactive: emailthis(message)
            break
        time.sleep(restartinterval)
    if not service_running(servicename):
        message = "Service " + servicename + " can\'t be started after " + str(restartattemptcount) + " attempts"
        if emailactive: emailthis(message)
        logging.warning(message)
        time.sleep(checkinterval)