API tools faq
paste
rs232

watchdog

rs232
Feb 9th, 2022 (edited)
197
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 14.08 KB | None | 0 0
raw download clone embed print report
  1. #!/bin/sh
  2. # Initial release: 2015 shibby
  3. # changes/fixes: 2018-2021 - pedro
  4. # v1.023 revised: 2022 - rs232
  5.  
  6. PID=$$
  7. PIDFILE="/var/run/watchdog.pid"
  8. IPLISTFILE="/tmp/watchdog.iplist"
  9. MWAN=$(nvram get mwan_num)
  10. IPLIST=""
  11. MWANTABLE="wan"
  12. white="\033[1;37m"
  13. black="\033[0;40m"
  14. green="\e[30m\033[42m"
  15. red="\e[0;30m\033[41m"
  16. now=$(date +%s)
  17.  
  18. echo -e "${white}
  19. ────────────  FreshTomato WatchDog  ────────────${black}
  20.  
  21. ${white}── Current default gateway/s ───────────────────${black}
  22. $(ip r | grep default)
  23.  
  24. ${white}── Probing ─────────────────────────────────────${black}"
  25. i=1
  26. while [ $i -le $MWAN ]; do
  27.     [ "$i" -gt 1 ] && MWANTABLE="$MWANTABLE wan$i"
  28.     i=$((i+1))
  29. done
  30.  
  31. LOGS="logger -t watchdog[$PID]"
  32. [ "$(nvram get mwan_debug)" -gt 0 ] && DEBUG="logger -p DEBUG -t watchdog[$PID] *** " || DEBUG="echo -e"
  33.  
  34. timeout() {
  35.     local cmd_pid sleep_pid retval
  36.     (shift; "$@") &
  37.     cmd_pid=$!
  38.     (sleep "$1"; kill "$cmd_pid" 2>/dev/null) &
  39.     sleep_pid=$!
  40.     wait "$cmd_pid"
  41.     retval=$?
  42.     kill "$sleep_pid" 2>/dev/null
  43.     return "$retval"
  44. }
  45.  
  46. findHost() {
  47.     local host ip
  48.     local dst=$(nvram get mwan_ckdst)
  49.     local hostlist=$(echo $dst | sed 's/,/ /')
  50.  
  51.     for host in $hostlist; do
  52.         echo $host | grep -Eo '((([a-zA-Z]{1,2})|([0-9]{1,2})|([a-zA-Z0-9]{1,2})|([a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9]))\.)+[a-zA-Z]{2,6}' && {
  53.         ip=$(nslookup $host 127.0.0.1 2>/dev/null | tail +5 | grep -v :: | awk '{print $3} | tail -1')
  54.         IPLIST="${IPLIST} $ip"
  55.         } || IPLIST="${IPLIST} $host"
  56.     done
  57.  
  58.     echo "$IPLIST" > $IPLISTFILE # because we're in subshell
  59. }
  60.  
  61. dhcpFix() {
  62.     dhcpc-release $PREFIX
  63.     sleep 1
  64.     dhcpc-renew $PREFIX
  65. }
  66.  
  67. ipCheck() {
  68.     [ "$PROTO" == "dhcp" -a $(ifconfig $IFACE | grep inet | grep -Eo '((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' | wc -l ) -eq 0 ] && { dhcpFix $PREFIX; sleep 3; }
  69.     [ $(ifconfig $IFACE | grep inet | grep -Eo '((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' | wc -l ) -eq 0 ] && {
  70.         $DEBUG "$PREFIX doesn't have an IP address. Skipping..."
  71.         echo "2" >> $STATE_FILE
  72.         echo "$now 2" >> $STATE_FILE_LOG
  73.         continue;
  74.     }
  75. }
  76.  
  77. resultsOut() {
  78.     for PREFIX in $MWANTABLE; do
  79.         STATE_FILE_LOG="/var/lib/misc/"$PREFIX"_state_log"
  80.         echo -e "───── ${red}$PREFIX ─ Last 10 results${black} (since boot) ─────"
  81.         cat $STATE_FILE_LOG | tail -10 | awk '{print strftime("%c",$0)}{print $2}'
  82.         echo ""
  83.     done
  84. exit   
  85.     }
  86.  
  87. resultsOutDown() {
  88.     for PREFIX in $MWANTABLE; do
  89.         STATE_FILE_LOG="/var/lib/misc/"$PREFIX"_state_log"
  90.         echo -e "───── ${red}$PREFIX ─ Last 10 failures${black} (since boot) ─────"
  91.         cat $STATE_FILE_LOG | grep -E '\ 0$' | tail -10 | awk '{print strftime("%c",$0)}{print $2}'
  92.         echo ""
  93.     done
  94. exit
  95.     }
  96.  
  97. # loadBalancing=0
  98. # for PREFIX in $MWANTABLE; do
  99.     # WEIGHT=$(nvram get "$PREFIX"_weight)
  100.     # [ $WEIGHT -eq 0 ] && continue || loadBalancing="$((loadBalancing+1))"
  101. # done
  102. # [ $loadBalancing -ge 2 ] && {
  103. # routing="ip route add default scope global "
  104. # for PREFIX in $MWANTABLE; do
  105. # # GW=$(nvram get "$PREFIX"_gateway)
  106.     # IFACE=$(nvram get "$PREFIX"_iface)
  107.     # WEIGHT=$(nvram get "$PREFIX"_weight)
  108.     # routing="${routing} nexthop dev $IFACE weight $WEIGHT "  
  109. # done
  110. # }
  111.  
  112. watchdogRun() {
  113.     for PREFIX in $MWANTABLE; do
  114.         IFACE=$(nvram get "$PREFIX"_iface) #eth1, vlan2, etc
  115.         ISPPPD=$([ -f /tmp/ppp/pppd$PREFIX ] && echo 1 || echo 0)
  116.         WEIGHT=$(nvram get "$PREFIX"_weight)
  117.         METHOD=$(nvram get "$PREFIX"_ckmtd)
  118.         PROTO=$(nvram get "$PREFIX"_proto)
  119.         DEMAND=$(nvram get "$PREFIX"_ppp_demand)
  120.         RESULT=0
  121.         # PREFIX_MWAN=$PREFIX  #wan, wan2, etc
  122.         STATE_FILE="/var/lib/misc/"$PREFIX"_state"
  123.         STATE_FILE_LOG="/var/lib/misc/"$PREFIX"_state_log"
  124.  
  125.         [ "$(nvram get "$PREFIX"_ck_pause)" -eq 1 ] && {
  126.             $DEBUG "${white}Watchdog paused for $PREFIX - skipping ...${black}"
  127.             continue
  128.         }
  129.  
  130.         cp $STATE_FILE "$STATE_FILE"_pre 2>/dev/null
  131.  
  132.         [ "$PROTO" != "disabled" ] && {
  133.             [ "$(nvram get mwan_debug)" -gt 0 ] && {
  134.                 ISUP=$(wanuptime "$PREFIX")
  135.                 ISGW=$(ip route | grep $IFACE | grep -v "link" | wc -l)
  136.                 $DEBUG "prefix=$PREFIX, iface=$IFACE, uptime=$ISUP, ISGW=$ISGW, WEIGHT=$WEIGHT"
  137.             }
  138.  
  139.             sed -i -e :a -e  '$q;N;1000,$D;ba' $STATE_FILE_LOG
  140.  
  141.             ipCheck
  142.  
  143.             DEFAULT_ROUTE_FRAGMENT=$(ip route | grep default | cut -d' ' -f2-)
  144.             GATEWAY="$(nvram get "$PREFIX"_gateway)"
  145.             [ "$ISPPPD" -eq 1 ] && GATEWAY_FRAGMENT=""
  146.            
  147.             $DEBUG "${white}>─ Testing $(printf "%4s%-s\n" $PREFIX) ─ Weight: $(printf "%2s%-s\n" $WEIGHT) ──────────────────<${black}"
  148.             [ -s $STATE_FILE_LOG ] && echo -e "Previous test results > $(tail -23 $STATE_FILE_LOG | awk '{print $2}' | tr -d "\n" |  awk '{ for (i=NF; i>1; i--) printf("%s ",$i); print $1; }')"
  149.             # [ -z "$DEFAULT_ROUTE_FRAGMENT" ] && { #default gateway missing...??? I'll add it manually.
  150.                 # echo "shit"
  151.                 # for PREFIX in $MWANTABLE; do
  152.                     # [ $(nvram get "$PREFIX"_weight) -ge 1 ] && {
  153.                         # ip route add default via $GATEWAY dev $IFACE
  154.                         # # echo "ip route add default dev $IFACE"
  155.                         # break
  156.                         # }
  157.                 # done
  158.             # }
  159.  
  160.  
  161.             for IP in $IPLIST; do
  162.  
  163.                 # ipCheck
  164.                
  165.                 [ ! -z "$DEFAULT_ROUTE_FRAGMENT" ] && {
  166.                     ROUTE_EXEC="ip route add $IP/32 via $GATEWAY dev $IFACE"
  167.                     # $DEBUG "a. Injecting: $ROUTE_EXEC"
  168.                     $ROUTE_EXEC 2>/dev/null
  169.                 }
  170.                 ROUTE_EXEC="ip route add $IP dev $IFACE $GATEWAY_FRAGMENT metric 50000"
  171.                 # $DEBUG "a. Injecting: $ROUTE_EXEC"
  172.                 $ROUTE_EXEC 2>/dev/null
  173.             done
  174.  
  175.  
  176.             if [ "$METHOD" -eq 1 ]; then
  177.                 ckping
  178.             elif [ "$METHOD" -eq 2 ]; then
  179.                 cktracert
  180.             else
  181.                 ckcurl
  182.             fi 
  183.            
  184.             for IP in $IPLIST; do
  185.  
  186.                 [ ! -z "$DEFAULT_ROUTE_FRAGMENT" ] && {
  187.                     ROUTE_EXEC="ip route del $IP/32 via $GATEWAY dev $IFACE"
  188.                     # $DEBUG "c. Removing: $ROUTE_EXEC"
  189.                     $ROUTE_EXEC 2>/dev/null
  190.                 }
  191.                 ROUTE_EXEC="ip route del $IP dev $IFACE $GATEWAY_FRAGMENT metric 50000"
  192.                 # $DEBUG "c. Removing: $ROUTE_EXEC"
  193.                 $ROUTE_EXEC 2>/dev/null
  194.             done
  195.  
  196.             # wan is down
  197.             [ "$RESULT" -eq 0 ] &&  {
  198.                 $DEBUG "${red}>>> Connection $PREFIX is down! <<<${black}
  199.                 "
  200.                 [ "$PROTO" == "lte" ] && {
  201.                     $LOGS "Connection $PREFIX DOWN - Reconnecting ..."
  202.                     echo "0" > $STATE_FILE
  203.                     echo "$now 0" >> $STATE_FILE_LOG
  204.                     switch4g $PREFIX
  205.                 } || {
  206.                     [ "$PREFIX" == "wan" -a "$MWAN" -gt 1 ] && PREFIX_MWAN="wan1" # "wan" means restart all WANs, but we only want restart one
  207.  
  208.                     [ "$(nvram get action_service)" == "wan-restart" -o "$(nvram get action_service)" == $PREFIX"-restart" -o "$(nvram get action_service)" == "wan-restart-c" -o "$(nvram get action_service)" == $PREFIX"-restart-c" ] && {
  209.                         $LOGS "Connection $PREFIX DOWN - Reconnect is already in progress ..."
  210.                     } || {
  211.                         echo "0" > $STATE_FILE
  212.                         echo "$now 0" >> $STATE_FILE_LOG
  213.  
  214.                         if [ "$PROTO" == "pppoe" -o "$PROTO" == "pptp" -o "$PROTO" == "l2tp" -o "$PROTO" == "ppp3g" ] && [ "$DEMAND" -eq 1 -a "$ISPPPD" -eq 0 ]; then
  215.                             $LOGS "Killing orphaned connect-on-demand listen process ..."
  216.                             LISTEN_PID=$(ps | grep listen | grep $PREFIX | awk '{print $1}' | head -n1)
  217.                             [ -n $LISTEN_PID ] && {
  218.                                 kill -9 $LISTEN_PID
  219.                                 $LOGS "Killed $LISTEN_PID"
  220.                             } || {
  221.                                 $LOGS "Connect-on-demand listen not running"
  222.                             }
  223.  
  224.                             $LOGS "Connection $PREFIX DOWN - Reconnecting ..."
  225.                             service $PREFIX restart
  226.                         else
  227.                             $LOGS "Connection $PREFIX DOWN - Reconnect will be handled by another process ..."
  228.                         fi
  229.                     }
  230.                 }
  231.             } || {
  232.                 [ "$PROTO" == "dhcp" -a "$(cat $STATE_FILE)" -eq 0 ] && { # connected + DHCP + previous status - disconnected? release/renew
  233.                 dhcpFix
  234.                 }
  235.                 $DEBUG "${green}<<< Connection $PREFIX is up! >>>${black}
  236.                 "
  237.                 echo "1" > $STATE_FILE
  238.                 echo "$now 1" >> $STATE_FILE_LOG
  239.             }
  240.         }
  241.     done
  242. }
  243.  
  244. cktracert() {
  245.     local RXBYTES1=$(cat /sys/class/net/$IFACE/statistics/rx_bytes)
  246.     local RXBYTES2 IP
  247.  
  248.     # $DEBUG "a. ${white}traceroute${black} test for $PREFIX ($IFACE):."
  249.     $DEBUG "a.traceroute -i $IFACE -n -w 1 -m 5 -q 1 -z $IP"
  250.     for IP in $IPLIST; do
  251.         # we need only send/receive few packages to be sure is connection works.
  252.         traceroute -i $IFACE -n -w 1 -m 4 -q 1 -z 1 $IP > /dev/null 2>&1
  253.         usleep 200
  254.     done
  255.  
  256.     RXBYTES2=$(cat /sys/class/net/$IFACE/statistics/rx_bytes)
  257.  
  258.     [ "$RXBYTES2" -gt "$RXBYTES1" ] && RESULT=1
  259.     $DEBUG "b. $traceroute for $IFACE: RX2=$RXBYTES2 RX1=$RXBYTES1"
  260.     [ "$RESULT" -gt 0 ] && $DEBUG "c. Test result: ${white}PASSED${black}" || $DEBUG "c. Test result: ${white}!! FAILED !!${black}"
  261. }
  262.  
  263. ckping() {
  264.     local IP CHECK
  265.  
  266.     $DEBUG "a. ${white}ping${black} test for $PREFIX ($IFACE):"
  267.     i=0
  268.     for IP in $IPLIST; do
  269.         #$DEBUG "  b. ping -c 2 -A -w 4 -q -I $IFACE $IP"
  270.         # "0" means 100% loss - not receive any package
  271.         i=$((i+1))
  272.         ping -c $((i+1)) -A -W $((i+2)) -q -I $IFACE $IP >/dev/null && RESULT=$((RESULT+1))
  273.         $DEBUG "b${i}. ping -c $((i+1)) -A -W $((i+2)) -q -I $IFACE $IP == $RESULT"
  274.         [ "$RESULT" -gt 0 ] && break;
  275.         sleep $i
  276.     done
  277.  
  278.     [ "$RESULT" -gt 0 ] && $DEBUG "c. Test result: ${white}PASSED${black}" || $DEBUG "c. Test result: ${white}!! FAILED !!${black}"
  279. }
  280.  
  281. ckcurl() {
  282.     local IP CHECK
  283.  
  284.     $DEBUG "a. ${white}curl${black} test for $PREFIX ($IFACE):"
  285.     i=0
  286.     for IP in $IPLIST; do
  287.         # $DEBUG "  b. curl $IP --interface $IFACE --connect-timeout 5 -ksfI"
  288.         i=$((i+1))
  289.         curl $IP --interface $IFACE --connect-timeout $((i+3)) -ksfI -o /dev/null && RESULT=$((RESULT+1))
  290.         $DEBUG "b.${i} curl $IP --interface $IFACE --connect-timeout $((i+3)) -ksfI == $RESULT"
  291.         [ $RESULT -gt 0 ] && break
  292.         sleep $i
  293.     done
  294.  
  295.     [ "$RESULT" -gt 0 ] && $DEBUG "c. Test result: ${white}PASSED${black}" || $DEBUG "c. Test result: ${white}!! FAILED !!${black}"
  296. }
  297.  
  298. watchdogAdd() {
  299.     local CKTIME=$(nvram get mwan_cktime)
  300.     local MINS=$((CKTIME/60))
  301.  
  302.     [ "$MINS" -gt 0 ] && {
  303.         cru l | grep watchdogJob >/dev/null && cru a watchdogJob "*/$MINS * * * * /usr/sbin/watchdog"
  304.     }
  305. }
  306.  
  307. watchdogDel() {
  308.     cru l | grep watchdogJob >/dev/null && cru d watchdogJob
  309. }
  310.  
  311. mwanJob() {
  312.     cru l | grep mwanJob >/dev/null  && cru d mwanJob || cru a mwanJob "*/1 * * * * /usr/sbin/watchdog alive"
  313. }
  314.  
  315. mwanAlive() {
  316.     [ "$MWAN" -gt 1 ] && {
  317.         ps | grep [m]wanroute >/dev/null && $DEBUG "mwanroute is running" || {
  318.             $LOGS "mwanroute not found, launch process"
  319.             mwanroute
  320.         }
  321.     }
  322. }
  323.  
  324. checkPid() {
  325.     local PIDNO
  326.  
  327.     [ -f $PIDFILE ] && {
  328.         PIDNO=$(cat $PIDFILE)
  329.         cat "/proc/$PIDNO/cmdline" > /dev/null 2>&1
  330.  
  331.         [ $? -eq 0 ] && {
  332.             $LOGS "Another process in action - Exiting ..."
  333.             exit 0
  334.         } || {
  335.             # Process not found assume not running
  336.             echo $PID > $PIDFILE
  337.             [ $? -ne 0 ] && {
  338.                 $LOGS "Could not create PID file"
  339.                 exit 0
  340.             }
  341.         }
  342.     } || {
  343.         echo $PID > $PIDFILE
  344.         [ $? -ne 0 ] && {
  345.             $LOGS "Could not create PID file"
  346.             exit 0
  347.         }
  348.     }
  349. }
  350.  
  351. checkPidSwitch() {
  352.     local SPREFIX
  353.  
  354.     for SPREFIX in $MWANTABLE; do
  355.         [ -f /var/run/switch3g_$SPREFIX.pid ] && {
  356.             ps | grep [s]witch3g && {
  357.                 $LOGS "Switch3g ($SPREFIX) script in action - Exiting ..."
  358.                 rm -f $PIDFILE > /dev/null 2>&1
  359.                 exit 0         
  360.             } || {
  361.                 # pid file exists but process doesn't
  362.                 rm /var/run/switch3g_$SPREFIX.pid
  363.             }
  364.         }
  365.  
  366.         [ -f /var/run/switch4g_$SPREFIX.pid ] && {
  367.             ps | grep [s]witch4g && {
  368.                 $LOGS "Switch4g ($SPREFIX) script in action - Exiting ..."
  369.                 rm -f $PIDFILE > /dev/null 2>&1
  370.                 exit 0
  371.             } || {
  372.                 # pid file exists but process doesn't
  373.                 rm /var/run/switch4g_$SPREFIX.pid
  374.                 }
  375.         }
  376.     done
  377. }
  378.  
  379. helpOut() {
  380.  
  381. echo -e "    List of supported parameters:
  382.  
  383.        ${white}help${black} = this screen
  384.      ${white}status${black} = Last 10 recorded resultson each wan
  385. ${white}status-down${black} = Last 10 failures on each wan
  386.  
  387. ${white}────────────────────────────────────────────────${black}
  388. "
  389. exit
  390.  }
  391.  
  392. ###################################################
  393.  
  394.  
  395. if [ "$1" == "add" ]; then
  396.     watchdogAdd
  397.     mwanJob
  398. elif [ "$1" == "del" ]; then
  399.     watchdogDel
  400. elif [ "$1" == "alive" ]; then
  401.     mwanAlive
  402. elif [ "$1" == "test" ]; then
  403.     watchdogRun
  404. elif [ "$1" == "status" ]; then
  405.     resultsOut
  406. elif [ "$1" == "status-down" ]; then
  407.     resultsOutDown 
  408. elif [ "$1" == "help" ]; then
  409.     helpOut
  410. else
  411.     checkPid
  412.  
  413.     checkPidSwitch
  414.  
  415.     mwanJob
  416.  
  417.     # run with a 10 sec timeout to not hang
  418.     timeout 10 findHost
  419.     [ -f $IPLISTFILE ] && IPLIST=$(cat $IPLISTFILE)
  420.     [ -z "$IPLIST" ] && IPLIST="192.0.43.10 4.2.2.4 139.130.4.5 198.153.194.2" # resilient IP if the list is empty
  421.     for IP in $IPLIST; do
  422.         for dns in $(nvram show 2>/dev/null | grep -E ^wan.?_dns= | cut -f2 -d= | tr "\n" " " | grep -Ev ^$); do
  423.         [ $dns == $IP ] && BADIPLIST="${BADIPLIST} $IP"
  424.         done
  425.     done
  426.     [[ ! -v ${BADIPLIST} ]] && {
  427.         IIPLIST=$(echo $IPLIST | tr " " "\n" | for i in $BADIPLIST; do grep -v $i; done | tr "\n" " " | tr -s ' ')
  428.         IPLIST=$IIPLIST
  429.         }
  430.     echo " TEST IPs =$IPLIST
  431.     "
  432.    
  433.     watchdogRun
  434.    
  435.     [ -f $IPLISTFILE ] && rm $IPLISTFILE
  436. fi
  437. echo -e "${white}────────────────────────────────────────────────${black}"
  438.  
  439. # Recivery (if needed#
  440. for PREFIX in $MWANTABLE; do
  441.     STATE_FILE="/var/lib/misc/"$PREFIX"_state"
  442.     [ $(cat "$STATE_FILE" 2>/dev/null) != $(cat "$STATE_FILE"_pre 2>/dev/null) ] && MODIFIED=$((MODIFIED+1));
  443. done
  444. [ ! -z ${MODIFIED} ] && {
  445. [ $(nvram get tinc_wanup) -eq 1 ] && {
  446.     echo -e "${white} != Change of status detected! !=${black} Performing recovery actions..."
  447.     sleep 10
  448.     ( ps w | grep -E [t]inc && service tinc restart ; ) >/dev/null 2>&1
  449.     ( ps w | grep -E [v]pn | awk '{print $5}' | cut -d/ -f4 | while read line; do service $line restart; done; ) > /dev/null 2>&1
  450.     }
  451. } || echo -e " ${white}== No changes ==${black} Same status as last run...";
  452. echo -e "${white}────────────────────────────────────────────────${black}
  453. "
  454. rm /var/lib/misc/*_pre > /dev/null 2>&1
  455. rm -f $PIDFILE > /dev/null 2>&1
  456.  
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!