#!/bin/bash
######################################################
# Program:      checkThreadpool.sh
# Date Created: 20 July 2012
# Date Updated: 1 Aug 2012
#               |_updated alert logic to combined value of 10K
#               |_Updated getValue function to work with both 2.7.0.5 && 2.8.0.4
#               |_Added log class to logging function, could probably be prettier though
# Developer:    @MonteVonB (Support Engineer)
# Description:  Checks the manager threadpool (private and public).
#               |_If either rises above 1000 for 5 consecutive passes, an alert is triggered
######################################################

#Variable definitions

#Array which contains managers to check. Format is (last octet of VIP|Short description of manager)
managerInfo=("6|Man1" "11|Man2" "21|Man3")
#Directory to place logs
logDir="/opt/Redacted/utils/alertScripts/alertLogs"
#name for logs
logName="thread.pool.log"
#Threshold for triggering
threshSize=10000

#Simple logging function. TODO: make this better
#First argument is text to print to log (must be wrapped in quotes"
#Second argument used to name log file
#Third argument is for log class (info,warning,error)
logGener ()
  {
    printf "$(date "+%m/%d/%y %H:%M.%S")\t$3\t$1\n" >> ${logDir}/${2}.${logName}
  }

#Function that uses curl to grab values from manager API
getValues ()
  {
    curl --silent "http://10.265.19.${1}:5929/command/monitor_service?id=THREAD_POOL"|grep -A3 "<pool_name>A_COMP</pool_name>"|awk -F"[><]" '/queue/{print $2,$3 }'
  }


#Function that contains the main processing
#Loops through all defined managers in the managerInfo variable
#If either threadpool exceeds defined threshSize, iterate up dot file
#If neither threadpool exceeds defined threshSize, clear out dot file
#If dot file value equals 5, trigger alert, and clear dot file
#Alert that is generated is specific to system which caused it, and contains relevant log lines
#TODO: seperate alert triggering into separate function
doWork ()
  {
    for manInfo in "${managerInfo[@]}";do
      rawData=($(getValues ${manInfo%%|*}))
      publicQueuesize="${rawData[1]}"
      privateQueuesize="${rawData[3]}"
        logGener "Public Queue Size is ${publicQueuesize}" "${manInfo##*|}" "Info"
        logGener "Private Queue Size is ${privateQueuesize}" "${manInfo##*|}" "Info"
          if (( "$((${privateQueuesize} +  ${publicQueuesize}))" > "${threshSize}" ));then
            if [[ ! -s "${logDir}/.${manInfo##*|}" ]];then
                logGener "Combined Threadpool size has exceeded ${threshSize}" "${manInfo##*|}" "Warning"
                echo "1" > "${logDir}/.${manInfo##*|}"
            else
                let numCount=$(cat "${logDir}/.${manInfo##*|}")+1
                  if (( ${numCount} >= 5 ));then
                    printf "The threadpool in ${manInfo##*|} has exceeded ${threshSize} 5 times in the last 5 minutes.\n\n$(tail -15 ${logDir}/${manInfo##*|}.${logName})\n"|ssh redacted@172.265.129.36 "mail -s \"Threadpool alert in ${manInfo##*|}\" example@redacted.com"
                    > "${logDir}/.${manInfo##*|}"
                    logGener "Combined Threadpool size has exceeded ${threshSize} 5 times in 5 minutes. Sending Email alert" "${manInfo##*|}" "Error"
                  else
                    echo ${numCount} > "${logDir}/.${manInfo##*|}"
                    logGener "Combined Threadpool size has exceeded ${threshSize} ${numCount} times." "${manInfo##*|}" "Warning"
                  fi
            fi
          else
            if [[  -s "${logDir}/.${manInfo##*|}" ]];then
                > "${logDir}/.${manInfo##*|}"
            fi
          fi
    done
  }


#Call function to do work
doWork