Pi-hole state probe.

#!/bin/bash

# target IP of monitored Pi-hole
target=192.168.1.3

# count the pings
count=$( ping -c 3 -w 3 $target | grep icmp* | wc -l )

# folder where we store files
dir=/home/pi/

#check for active state
probe2=$(curl -s http://${target}/admin/ | grep Active)
check2=$(echo $probe2)

#check for offline state
probe=$(curl -s http://${target}/admin/ | grep offline)
check=$(echo $probe)

#run selftest (this is on the Pi-hole instance that will take over in case the main one is offline/in error state)
selftest=$(curl -s http://127.0.0.1/admin/ | grep "Active")
selfecho=$(echo $selftest)

function selftest {
self=$(curl -s http://127.0.0.1/admin/ | grep "Active")
selfecho=$(echo $self)
}

function pass {
sudo cp /home/pi/stuff/*.* /etc/dnsmasq.d/
}

#notifications via pushover (will send process state via pushover and attempt restart)
function ernotify {
echo "ER1 - Moving resolv.conf file"
sudo mv /etc/resolv.conf /etc/resolv.conf.old
echo "ER1 - Using 1.1.1.1 as DNS for notification"
sudo echo "nameserver 1.1.1.1" > /etc/resolv.conf
echo "ER1 - Fail Notification"
pushover pihole "<b><font color="red"><b>Pi-Hole FTLS DNS is in error state</font> since $(date). <br>Attempting restart</b>"
echo "ER1 - Restoring resolv.conf"
sudo mv /etc/resolv.conf.old /etc/resolv.conf
echo "ER1 - Grabbing FTL status"
selfstatus1=$(sudo systemctl status pihole-FTL.service > ${dir}FTLstatusER1)
selffail31=$(cat ${dir}FTLstatusER1 | tail -3 | awk 'NR==3')
selffail21=$(cat ${dir}FTLstatusER1 | tail -3 | awk 'NR==2')
selffail11=$(cat ${dir}FTLstatusER1 | tail -3 | awk 'NR==1')
echo "ER1 - Restarting FTL"
sudo systemctl restart pihole-FTL.service
echo "ER1 - Successful notification"
pushover pihole "<b>Pi-Hole FTLS DNS is <font color="lime">fixed</b></font> (restart successful)<br> FTL-Service status prior to restart: <br><br>$selffail31 <br><br>$selffail21 <br><br>$selffail11 "
}

#aditional health check. if first attempt didn't fix the service, send a notification about it and request manual intervention
function ernotify2 {
echo "ER2 - Moving resolv.conf file"
sudo mv /etc/resolv.conf /etc/resolv.conf.old
echo "ER2 - Using 1.1.1.1 as DNS for notification"
sudo echo "nameserver 1.1.1.1" > /etc/resolv.conf
echo "ER2 - Grabbing FTL status"
selfstatus=$(sudo systemctl status pihole-FTL.service > ${dir}FTLstatusER2)
selffail3=$(cat ${dir}FTLstatusER2 | tail -3 | awk 'NR==3')
selffail2=$(cat ${dir}FTLstatusER2 | tail -3 | awk 'NR==2')
selffail1=$(cat ${dir}FTLstatusER2 | tail -3 | awk 'NR==1')
echo "ER2 - FTL Debug Notification sent"
pushover pihole "<b><font color="red"><b>Pi-hole FTL restart failed.</font></b> <br>Manual intervention is needed. FTL-Service output: <br><br>$selffail3 <br><br>$selffail2 <br><br>$selffail1 "
sleep 5
echo "ER2 - Restoring resolv.conf"
sudo mv /etc/resolv.conf.old /etc/resolv.conf
}

selftest

#compare/rpobe for error status
if [ -z "$selfecho" ]
then
    if [ -e ${dir}FTL.err ]
    then
    echo "FTL error file present. Exiting..."
    exit 0
    else
    echo ""
    echo "Local FTL in error state."
    echo ""
    ernotify
    selftest=$(curl -s http://127.0.0.1/admin/ | grep "Active")
    selfecho=$(echo $selftest)
    echo ""
    echo "Running additional check ..."
    if [ -z "$selfecho" ]
    then
    echo ""
    echo "FTL still in error mode. Running diagnostic check..."
    echo ""
    ernotify2
    echo ""
    echo "Create lock file..."
    touch ${dir}FTL.err
    echo ""
    else
    echo ""
    echo "FTL restart - Second check. FTL up and running."
    fi
    fi
else
    echo ""
    echo "Local FTL up and running."
    echo ""
    sudo rm -f ${dir}FTL.err
fi


#check for reply
if [ -z "$check" ] && [ -z "$check2" ]
then
        echo "MezelPi is not responding!"
        if [ -e ${dir}dhcp.on ]
        then
        echo "DHCP server already enabled. No changes or notifications performed."
        exit 0
else
        echo "Generating lock file"
        touch ${dir}dhcp.on
        echo "Done."
#        sudo pihole -a enabledhcp "192.168.1.2" "192.168.1.110" "192.168.1.1" "1h" "local"
        sudo pihole -a enabledhcp "192.168.1.2" "192.168.1.110" "192.168.1.1" "1" "local"
        sudo rm -f /etc/dnsmasq.d/02-pihole-dhcp.conf
        sudo rm -f /etc/dnsmasq.d/03-pihole-wildcard.conf
        sudo rm -f /etc/dnsmasq.d/04-pihole-static-dhcp.conf
        sudo rm -f /etc/dnsmasq.d/05-3-dns.conf
        fi

#check for 04-pihole-static-dhcp.conf
if [ -e /etc/dnsmasq.d/04-pihole-static-dhcp.conf ]
then
    echo "04-pihole-static-dhcp.conf present. No changes made."
else
    echo "04-pihole-static-dhcp.conf missing. Generating file."

#pass strings in order for existing dhcp clients to keep IPs
pass
echo "Done."

#restart dns and activate DHCP server
echo "Restarting DNS."
#pihole restartdns
sleep 2
sudo pkill pihole-FTL
sleep 2
sudo systemctl restart pihole-FTL.service
echo "Done."

#notify
#sleep 10
(pushover pihole "MezelPi is not responding,  backup DHCP is<font color="red"><b> ON</font></b>." &)
echo "Notification sent."
fi

else
if [ -z "$check2" ] && [[ $check == *"offline"* ]]

    then
        echo "MezerlPi is up with errors!"
        if [ -e ${dir}dhcp.on ]
        then
        echo "DHCP server already enabled. No changes or notifications performed."
        exit 0
  else
        echo "Generating lock file"
        touch ${dir}dhcp.on
        echo "Done."
#        sudo pihole -a enabledhcp "192.168.1.200" "192.168.1.251" "192.168.1.1" "1h" "local"
#        sudo pihole -a enabledhcp "192.168.1.200" "192.168.1.251" "192.168.1.1" "1" "local"
        sudo pihole -a enabledhcp "192.168.1.2" "192.168.1.110" "192.168.1.1" "1" "local"
        sudo rm -f /etc/dnsmasq.d/02-pihole-dhcp.conf
        sudo rm -f /etc/dnsmasq.d/03-pihole-wildcard.conf
        sudo rm -f /etc/dnsmasq.d/04-pihole-static-dhcp.conf
        sudo rm -f /etc/dnsmasq.d/05-3-dns.conf
        fi

#check for 04-pihole-static-dhcp.conf
 if [ -e /etc/dnsmasq.d/04-pihole-static-dhcp.conf ]
then
    echo "04-pihole-static-dhcp.conf present. No changes made."
else
    echo "04-pihole-static-dhcp.conf missing. Generating file."

#copy config files in order for existing dhcp clients to keep IPs
pass
echo "Done."

#restart dns and activate DHCP server
echo "Restarting DNS."
sudo pkill pihole-FTL
sleep 2
sudo systemctl start pihole-FTL.service
echo "Done."
#send a pushover notification
(pushover pihole "MezelPi is in in error state(FTL offline), backup DHCP is<font color="red"><b> ON</font></b>." &)
echo "Notification sent."
exit 0
 fi
else
    echo "MezelPi is Alive!"

#remove DHCP settings.

 if [ -e /etc/dnsmasq.d/02-pihole-dhcp.conf ]
 then
echo "Removing settings."
sudo rm -f /etc/dnsmasq.d/02-pihole-dhcp.conf
sudo rm -f /etc/dnsmasq.d/03-pihole-wildcard.conf
sudo rm -f /etc/dnsmasq.d/04-pihole-static-dhcp.conf
sudo rm -f /etc/dnsmasq.d/dhcp-join.conf
sudo rm -f /etc/dnsmasq.d/05-3-dns.conf

#restart pihole / disable DNS seerver
echo "Restarting DNS."
sudo pihole -a disabledhcp
sleep 2
sudo pkill pihole-FTL
sleep 2
sudo systemctl start pihole-FTL.service
sudo rm -f ${dir}dhcp.on
echo "Done."

#send pushover notification
(pushover pihole "MezelPiHole backup DHCP is<font color="lime"><b> OFF</font></b>." &)
echo "Notification sent."
else
    echo "Lock file already deleted. No action performed."
 fi
 fi
fi