vrabie_mica network failover control script

#!/bin/bash

LOCKF=/run/isp-failover
if ! lockfile-create -r1 $LOCKF; then exit; fi

# REPS*WAIT must be < 60sec for once/min crontab execution
REPS=4
WAIT=14

WAN1=eth0.5
WAN2=eth0.4
# WAN2=wwan0_1

NO_REVERT=/tmp/no-revert
NO_SENIN=/tmp/no-senin

WAN1_GW=`</run/DGW`
WAN2_GW=`</run/LGW`

TEST_LIST="8.8.8.8 1.0.0.1 9.9.9.9 208.67.222.222 192.5.5.241 64.79.96.12 198.205.126.25"

# how many test points must be unreachable to declare an outage
TRIGGER=4

# probe fewer test points when on metered cellular
CELLBK_TEST="8.8.8.8 1.0.0.1 208.67.222.222"
CELLBK_TRIGGER=3
CELLBK_REVERT_DELAY=3

REVERT_NORMAL_MIN=10
REVERT_NORMAL_PPS=20
TRAFFIC_WINDOW=3

FAILOVER_TABLES="254 10 20 250"
FAILOVER_TUNNELS="tun25 tun11 tun41 tun3 tun1 tun0 tun6 tun2 tun4 tun8"  # priority order

CELLFAIL_TABLES="254 10 20"   # skip 250; never route Smart TVs via metered LTE
CELL_TUNNEL=tun3

PREFER_WAN1_TUNNELS="tun11 tun25 tun41 tun3 tun1 tun6 tun4 tun8"
PREFER_WAN2_TUNNELS="tun2 tun0"
PREFER_WAN1_TABLES="254 10 20"
PREFER_WAN2_TABLES="250"
PREFER_VOIP=$WAN1

PREFER_SENIN=$WAN2

OVPN=/etc/openvpn/imp
ON_CELLBK=/tmp/.on-lte
WAN1_DOWN=/run/.wan1-down
WAN2_DOWN=/run/.wan2-down
NORMAL=/run/.uplinks-normal
BACK_TO_NORMAL=/run/.uplinks-back
CUTOVER=/run/.wan-cutover

ASTCH=/tmp/_astch_ifl
DR=/usr/local/sbin/devregs
PWM2SET="/usr/local/sbin/imx6pwm -q 2"

LOGF=/var/log/isp-failover
TMPF=/tmp/ifl-$$

shopt -s expand_aliases
alias DT='date +"%a %Y-%m-%d %H:%M:%S"'

PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/sbin:/usr/local/bin

# local status indicator LED
function led2 () {
 $DR IOMUXC_SW_MUX_CTL_PAD_DISP0_DAT9 2  >/dev/null 2>&1
 $PWM2SET $1
}

function cleanexit () {
  lockfile-remove $LOCKF
  rm -f $TMPF
  exit 0
}

function rxpkts () {
  ifconfig $1 |grep "  RX packets:" |cut -d: -f2|cut -d' ' -f1
}

function senin_defroute () {
 if [[ ! -f $NO_SENIN ]]; then
  if fping -B1 -b12 senin; then
   if [[ "$1" = "$WAN2" ]]; then
     ssh senin defroute tmo
   else
     ssh senin defroute quasit
   fi
  fi
 fi
}

function revert_to_preferred_links () {
  echo "`DT` : revert to preferred links" | tee -a $LOGF

  for t in $PREFER_WAN1_TABLES; do
    R=`ip route show table $t | grep ^default`
    if echo $R | grep -q $WAN1; then
      echo "`DT` : skipping table $t - already on $WAN1"
    else
      C="ip route delete table $t $R"
      $C 2>/dev/null
      $C 2>/dev/null
      echo "`DT` : $C"
      C="ip route add table $t default via $WAN1_GW dev $WAN1"
      $C
      echo "`DT` : $C"
    fi
  done

  voip -$PREFER_VOIP

  for t in $PREFER_WAN2_TABLES; do
    R=`ip route show table $t | grep ^default`
    if echo $R | grep -q $WAN2; then
      echo "`DT` : skipping table $t - already on $WAN2"
    else
      C="ip route delete table $t $R"
      $C 2>/dev/null
      $C 2>/dev/null
      echo "`DT` : $C"
      C="ip route add table $t default via $WAN2_GW dev $WAN2"
      $C
      echo "`DT` : $C"
    fi
  done

  for t in $PREFER_WAN1_TUNNELS; do
    if ! $t |grep -q $WAN1; then
      echo "`DT` : revert $t to $WAN1"
      $t $WAN1
    else
      echo "skipping $t"
    fi
  done

  for t in $PREFER_WAN2_TUNNELS; do
    if ! $t |grep -q $WAN2; then
      echo "`DT` : revert $t to $WAN2"
      $t $WAN2
    else
      echo "skipping $t"
    fi
  done

  senin_defroute $PREFER_SENIN
}

function cutover_wan_to_wan () {
  echo "`DT` : cutover defaults: $1 to $2" | tee -a $LOGF
  for t in $FAILOVER_TABLES; do
    R=`ip route show table $t | grep ^default`
    if echo $R | grep -q $1; then
      C="ip route delete table $t $R"
      $C
      $C 2>/dev/null
      echo "`DT` : $C"
      if [[ "$2" == "$WAN1" ]]; then GW=$WAN1_GW; else GW=$WAN2_GW; fi
      C="ip route add table $t default via $GW dev $2"
      $C
      echo "`DT` : $C"
    elif [[ "$R" = "" ]]; then
      echo "`DT` : table $t - no default"
      if [[ "$2" == "$WAN1" ]]; then GW=$WAN1_GW; else GW=$WAN2_GW; fi
      C="ip route add table $t default via $GW dev $2"
      $C
      echo "`DT` : $C"
    else
      echo "`DT` : skipping table $t - no default to $1"
    fi
  done

  voip -$2

  echo "`DT` : cutover tunnels: $1 to $2" | tee -a $LOGF
  for t in $FAILOVER_TUNNELS; do
    if $t |grep -q $1; then
      echo "`DT` : $t to $2"
      $t $2
    elif ! $t |grep -q UGH; then
      echo "`DT` : $t no route - to $2"
      $t $2
    else
      echo "skipping $t"
    fi
  done

  senin_defroute $2
}


function cutover_cellbk_to_wan() {
  led2 30
  RF=/run/.wan${1}-back

  if [[ ! -f $RF ]]; then
    echo 0 >$RF
    echo "`DT` : WAN$1 restored? count 0" | tee -a $LOGF
    cleanexit
  fi

  TIME_BACK=`<$RF`
  if [[ $TIME_BACK -lt $CELLBK_REVERT_DELAY ]]; then
    TIME_BACK=$[ $TIME_BACK + 1]
    echo "$TIME_BACK" >$RF
    echo "`DT` : WAN$1 restored? count $TIME_BACK" | tee -a $LOGF
    cleanexit
  fi

  asterisk -rx "core show channels" |grep SIP\/ >$ASTCH
  if [[ -s $ASTCH ]]; then
    TIME_BACK=$[ $TIME_BACK + 1]
    echo "$TIME_BACK" >$RF
    echo "`DT` : WAN$1 restored - count $TIME_BACK - waiting on phone calls" | tee -a $LOGF
    cleanexit
  fi

  rm -rf $RF
  echo "`DT` : cutover defaults: Cellular-Backup to $1" | tee -a $LOGF
  for t in $FAILOVER_TABLES; do
    R=`ip route show table $t | grep ^default`
    if echo $R | grep -q ' lte'; then         # don't repoint tunneled defaults
      C="ip route delete table $t $R"
      $C
      $C 2>/dev/null
      echo "`DT` : $C"
      if [[ "$1" == "$WAN1" ]]; then GW=$WAN1_GW; else GW=$WAN2_GW; fi
      C="ip route add table $t default via $GW dev $1"
      $C
      echo "`DT` : $C"
    else
      echo "`DT` : skipping table $t - no cell-backup default"
    fi
  done

  voip -$1

  echo "`DT` : cutover tunnels: Cellular-Backup to $1" | tee -a $LOGF
  for t in $FAILOVER_TUNNELS; do
    if $t |grep -q " lte"; then
      echo "`DT` : $t to $1"
      $t $1
    else
      echo "skipping $t"
    fi
  done

  rm -f $ON_CELLBK

  # stop flashing cell-backup indicator LEDs (local & remote)
  $PWM2SET 0
  led 1 0

  senin_defroute $2
}


function cutover_to_cellbk () {
  LL=$1

  # start outage-warning LEDs flashing
  led2 16
  led 1 15

  TRIED=0
  LINKUP=0
  while [[ $LINKUP == 0 && $TRIED -lt 3 ]]; do
    if [[ $TRIES -ge 3 ]]; then
      echo "`DT` : NO USABLE BACKUP LINK" | tee -a $LOGF
      cleanexit               # give up for now; hope a WAN link returns
    fi
    if ! ip link show |grep -q lte$LL; then
      echo "`DT` : bringing up cellular link $LL" | tee -a $LOGF
      tether$LL up
    else
      echo "`DT` : cellular link $LL already up" | tee -a $LOGF
    fi

    if ! ip link show |grep -q lte$LL; then
      TRIED="$[ $TRIED + 1 ]"
      NEXT_CELL="$[ $LL + 1 ]"
      if [[ $NEXT_CELL -gt 3 || $NEXT_CELL -lt 2 ]]; then NEXT_CELL=1; fi
      echo "`DT` : CELL-${LL} UNUSABLE, TRYING CELL-${NEXT_CELL}" | tee -a $LOGF
      LL=$NEXT_CELL
    else
      LINKUP=1

      echo "`DT` : cutover tunnels to CELL-$LL" | tee -a $LOGF
      for t in $FAILOVER_TUNNELS; do
        if $t |grep -qe $WAN1 -e $WAN2; then
          echo "`DT` : $t to lte$LL"
          $t lte$LL
        else
          echo "skipping $t"
        fi
      done

      voip -lte$LL
      # voip $CELL_TUNNEL

      echo "`DT` : cutover defaults to cell-tunnel $CELL_TUNNEL" | tee -a $LOGF
      for t in $CELLFAIL_TABLES; do
        R=`ip route show table $t | grep ^default`
        if echo $R | grep -vqe " lte" -e " tun"; then
          C="ip route delete table $t $R"
          $C
          $C 2>/dev/null
          echo "`DT` : $C" | tee -a $LOGF
          C="ip route add table $t default dev $CELL_TUNNEL"
          $C
          echo "`DT` : $C" | tee -a $LOGF
        else
          echo "`DT` : skipping table $t" | tee -a $LOGF
        fi
      done

      echo $LL >$ON_CELLBK

      if [[ $LL -ge 2 ]]; then
        # flash LEDs more rapidly while on expensive/limited backup path
        $PWM2SET 31881 30 1
        led 1 1 188
      else
        # once/second LED flash while on cell-backup #1
        led2 1
        led 1 1
      fi

      senin_defroute quasit

      cleanexit     # exit until next invokation after bringing up cell link

    fi

  done
}


function test_run () {
  WAN1_UP=1
  WAN2_UP=1
  WAN1_WAS_DOWN=1
  WAN2_WAS_DOWN=1

  if ! fping -B1 -ub12 -I$WAN1 -S$WAN1IP $TEST_LIST >$TMPF 2>/dev/null; then
    UNREACH=`cat $TMPF |xargs`
    if [[ ! -f $WAN1_DOWN ]]; then  # avoid repetitive messages during outage
      echo "`DT` : down via $WAN1: $UNREACH" | tee -a $LOGF
      WAN1_WAS_DOWN=0
    fi
    if [[ `cat $TMPF | wc -l` -ge $TRIGGER ]]; then
      WAN1_UP=0
      echo "$UNREACH" > $WAN1_DOWN
    else
      led2 32
      rm -f $WAN1_DOWN
    fi
  else
    rm -f $WAN1_DOWN
  fi

  if ! ifconfig $WAN2 >/dev/null 2>&1; then
    WAN2_UP=0
    if [[ ! -f $WAN2_DOWN ]]; then  # avoid repetitive messages during outage
      echo "`DT` : $WAN2 interface missing!" | tee -a $LOGF
      WAN2_WAS_DOWN=0
    fi
    echo "$WAN2 interface missing!" > $WAN2_DOWN
  elif ! fping -B1 -ub12 -I$WAN2 -S$WAN2IP $TEST_LIST >$TMPF 2>/dev/null; then
    UNREACH=`cat $TMPF |xargs`
    if [[ ! -f $WAN2_DOWN ]]; then  # avoid repetitive messages during outage
      echo "`DT` : down via $WAN2: $UNREACH" | tee -a $LOGF
      WAN2_WAS_DOWN=0
    fi
    if [[ `cat $TMPF | wc -l` -ge $TRIGGER ]]; then
      WAN2_UP=0
      echo "$UNREACH" > $WAN2_DOWN
    else
      led2 32
      rm -f $WAN2_DOWN
    fi
  else
    rm -f $WAN2_DOWN
  fi

  if [[ -f $ON_CELLBK ]]; then
    if [[ $WAN1_UP == 1 ]]; then
      touch /run/.cutover-cell-to-wan1
      touch $CUTOVER
      cutover_cellbk_to_wan $WAN1
    elif [[ $WAN2_UP == 1 ]]; then
      touch /run/.cutover-cell-to-wan2
      touch $CUTOVER
      cutover_cellbk_to_wan $WAN2
    else
      rm -f $NORMAL $BACK_TO_NORMAL
      CELL_INTF=lte`cat $ON_CELLBK`
      if ! fping -B1 -ub12 -I$CELL_INTF $CELLBK_TEST >$TMPF 2>/dev/null; then
        UNREACH=`cat $TMPF |xargs`
        echo "`DT` : down via $CELLBK_INTF: $UNREACH" | tee -a $LOGF
        if [[ `cat $TMPF | wc -l` -ge $CELLBK_TRIGGER ]]; then
          NEXT_CELL="$[ $CELL_INTF + 1 ]"
          if [[ $NEXT_CELL -gt 3 || $NEXT_CELL -lt 2 ]]; then NEXT_CELL=1; fi
          echo "`DT` : CELL-${CELL_INTF} UNUSABLE, TRYING CELL-${NEXT_CELL}" | tee -a $LOGF
          cutover_to_cellbk $NEXT_CELL
        fi
      else
        # only test once per cron invokation while on cell-backup with good link
        cleanexit
      fi
    fi
  else
    if [[ $WAN1_UP == 0 && $WAN2_UP == 1 && $WAN1_WAS_DOWN == 0 ]]; then
      touch /run/.cutover-wan1-to-2
      touch $CUTOVER
      rm -f $NORMAL $BACK_TO_NORMAL
      cutover_wan_to_wan $WAN1 $WAN2
    elif [[ $WAN2_UP == 0 && $WAN1_UP == 1 && $WAN2_WAS_DOWN == 0 ]]; then
      touch /run/.cutover-wan2-to-1
      touch $CUTOVER
      rm -f $NORMAL $BACK_TO_NORMAL
      cutover_wan_to_wan $WAN2 $WAN1
    elif [[ $WAN1_UP == 0 && $WAN2_UP == 0 ]]; then
      touch /run/.cutover-to-cell
      touch $CUTOVER
      rm -f $NORMAL $BACK_TO_NORMAL
      cutover_to_cellbk 1
    else
      if [[ $WAN1_UP == 1 && $WAN2_UP == 1 ]]; then
        $PWM2SET 0
        led 1 0
        touch $NORMAL
        if [[ ! -f $BACK_TO_NORMAL ]]; then
          touch -d "+$REVERT_NORMAL_MIN minutes" $BACK_TO_NORMAL
        elif [[ -f $CUTOVER && $NORMAL -nt $BACK_TO_NORMAL && ! -f $NO_REVERT ]]; then
          asterisk -rx "core show channels" |grep SIP\/ >$ASTCH
          if [[ ! -s $ASTCH ]]; then
            W1_RX_A=`rxpkts $WAN1`
            W2_RX_A=`rxpkts $WAN2`
            sleep $TRAFFIC_WINDOW
            W1_RX_Z=`rxpkts $WAN1`
            W2_RX_Z=`rxpkts $WAN2`
            W1_RX_PKTS=$[ $W1_RX_Z - $W1_RX_A ];
            W2_RX_PKTS=$[ $W2_RX_Z - $W2_RX_A ];
            PPS_THR=$[ $REVERT_NORMAL_PPS * $TRAFFIC_WINDOW ];
            if [[ $W1_RX_PKTS -lt $PPS_THR && $W2_RX_PKTS -lt $PPS_THR ]]; then
              echo "W1_RX_PKTS=$W1_RX_PKTS & W2_RX_PKTS=$W2_RX_PKTS < PPS_THR = $PPS_THR"
              revert_to_preferred_links
              rm -f $CUTOVER
            else
              echo "waiting for low-traffic period. W1_RX_PKTS=$W1_RX_PKTS / W2_RX_PKTS=$W2_RX_PKTS "
            fi
          else
            echo "waiting on phone calls to revert"
          fi
        fi
      fi
    fi
  fi
}

# unused but retained for reference
function list_client_tunnels() {
  cd $OVPN
  for v in `ps auxww |grep -v grep |grep "openvpn --config" |\
   awk '{print $13}' |sed 's/.*\///'  | xargs grep -l ^tls-client`; do
     t=`grep "^dev " $v |head -n 1 |awk '{print $2}'`
     r=`grep ^remote $v |head -n 1|awk '{print $2}'`
     echo $v $r $t
  done
}


## MAIN ENTRY POINT

#if [[ -f /run/.fastprobe ]]; then
# REPS=$FAST_REPS
# WAIT=$FAST_WAIT
#fi

sleep 2

WAN1IP=`ip addr show dev $WAN1 |grep 'inet '  |grep -v 192.168.100 |head -n 1 | awk '{print $2}' |cut -d/ -f1`
if  [[ "$WAN1IP" = "" ]]; then WAN1IP=0.0.0.0; fi

WAN2IP=`ip addr show dev $WAN2 |grep 'inet '  |head -n 1 | awk '{print $2}' |cut -d/ -f1`
if  [[ "$WAN2IP" = "" ]]; then WAN1IP=0.0.0.0; fi

RUN=1
while true; do
 test_run
 RUN=$[ $RUN + 1 ];
 if [[ $RUN -gt $REPS ]]; then
   # date
   cleanexit
 fi
 sleep $WAIT
done