gpfs_snmp_failover

#!/bin/bash
# This script implements gpfs snmp monitoring failover by changing the defined collector node from a failed node to one of the available nodes.
# The failover scheme uses a callback mechanism triggered by a nodeLeave and the eventNode is its only parameter
# by Benny Chitambira 2012 bchitambira at gmail.com

failedNode=$1
defaultNode="quorum_node_1"
standbyNode="quorum_node_2"
activeNode=`/usr/lpp/mmfs/bin/mmlscluster | awk /snmp/'{print $4}'`

status1=`mmgetstate -N $defaultNode | awk /$defaultNode/'{ print $3 }'`
status2=`mmgetstate -N $standbyNode | awk /$standbyNode/'{ print $3 }'`

#if [[ -z "$activeNode" ]]
#then
#  /usr/lpp/mmfs/bin/mmchnode --snmp-agent  -N $defaultNode
#fi

if [[ "$status1" != "active" && "$status2" == "active" && $activeNode == $defaultNode* ]]
then
  echo "***** `date` : Node leave event for $failedNode" >> /callback/snmp_collector_failover.log
  echo "  Node $defaultNode is not active, status is: $status1" >> /callback/snmp_collector_failover.log
  echo "  Making node $standbyNode the SNMP collector" >> /callback/snmp_collector_failover.log
  /usr/lpp/mmfs/bin/mmchnode --nosnmp-agent  -N $defaultNode
  /usr/lpp/mmfs/bin/mmchnode --snmp-agent  -N $standbyNode
elif [[ "$status1" == "active" && "$status2" != "active" && $activeNode == $standbyNode* ]]
then
  echo "***** `date` : Node leave or join event for $failedNode" >> /callback/snmp_collector_failover.log
  echo "  Node $defaultNode is now $status1" >> /callback/snmp_collector_failover.log
  echo "  Making node $defaultNode the SNMP collector" >> /callback/snmp_collector_failover.log
  /usr/lpp/mmfs/bin/mmchnode --nosnmp-agent  -N $standbyNode
  /usr/lpp/mmfs/bin/mmchnode --snmp-agent  -N $defaultNode
elif [[ "$status1" != "active" && "$status2" != "active" ]]
then
  echo "***** `date` : Node leave event for $failedNode" >> /callback/snmp_collector_failover.log
  echo "  Both default and standby snmp collector nodes are down, nothing to do" >> /callback/snmp_collector_failover.log
else
  echo "***** `date` : The collector node $activeNode is active, nothing to do" >> /callback/snmp_collector_failover.log
fi