#!/bin/bash

# If the script has already restarted a scan within LOCKOUT_INTERVAL seconds, lock that
# filesystem out from any further automatic restarts.
LOCKOUT_INTERVAL=1800 #30 minutes

FS_FILE="/home/<redacted>/check_sf/filesystems.txt"
LRT_FILE="/home/<redacted>/check_sf/last_restart_times.txt"
LO_FILE="/home/<redacted>/check_sf/locked_out.txt"
LOGGING_FILE="/home/<redacted>/check_sf/check_sf.log"


declare -A FILESYSTEMS
declare -A LAST_RESTART_TIMES
declare -A LOCKED_OUT


LOG="Script running at `date`\n"
EMAIL=""

#
# Read state from filesystem
#

while IFS="=" read -r key value; do
        if [[ -n "$key" && -n "$value" ]]; then
                FILESYSTEMS["$key"]="$value"
        fi
done < $FS_FILE

while IFS="=" read -r key value; do
        if [[ -n "$key" && -n "$value" ]]; then
                LAST_RESTART_TIMES["$key"]="$value"
        fi
done < $LRT_FILE

while IFS="=" read -r key value; do
        if [[ -n "$key" && -n "$value" ]]; then
                LOCKED_OUT["$key"]="$value"
        fi
done < $LO_FILE


#
# Get current sf scan/monitor status
#

SCANS=`sf scan list --running`;

#
# See which scans have failed
#

for FS in "${!FILESYSTEMS[@]}"; do
        if [[ "$SCANS" != *"$FS"* ]]; then
                LR="${LAST_RESTART_TIMES["$FS"]}"
                CURTIME=`date +%s`
                LOCK="${LOCKED_OUT["$FS"]}"
                echo "Scan $FS is not listed"
                EMAIL="${EMAIL}""Scan $FS is not listed""\n"
                LOG="${LOG}""Scan $FS is not listed. "

                if [[ "$LOCK" == "0" ]]; then
                        echo "  Current time is:              $CURTIME"
                        EMAIL="${EMAIL}""  Current time is:              $CURTIME""\n"
                        LOG="${LOG}""Current time: $CURTIME. "

                        echo "  Last scripted restart was at: $LR ($((CURTIME - LR)) seconds ago)"
                        EMAIL="${EMAIL}""  Last scripted restart was at: $LR ($((CURTIME - LR)) seconds ago)""\n"
                        LOG="${LOG}""Last autorestart was at $LR ($((CURTIME - LR)) seconds ago). "

                        if [[ $((CURTIME - LR)) -gt $LOCKOUT_INTERVAL ]]; then
                                echo "  Restarting $FS scan"
                                EMAIL="${EMAIL}""  Restarting $FS scan""\n"
                                LOG="${LOG}""Restarting $FS scan.""\n"
                                LAST_RESTART_TIMES["$FS"]="$CURTIME"
                                X=`sf volume monitor start $FS`
                        else
                                echo "  Last restart was too recent; locking out restarts for $FS"
                                EMAIL="${EMAIL}""  Last restart was too recent; locking out restarts for $FS""\n"
                                LOG="${LOG}""Last restart was too recent; locking out restarts for $FS.""\n"
                                LOCKED_OUT["$FS"]="1"
                        fi
                else
                        echo "  Restarts for $FS are locked out"
                        EMAIL="${EMAIL}""  Restarts for $FS are locked out""\n"
                        LOG="${LOG}""  Restarts for $FS are locked out.""\n"
                fi
        fi
done


#
# Send/write logging outputs
#
if [[ ! -z "$EMAIL" ]]; then
        echo -e $EMAIL | mail -s "check_sf cron output" <redacted>
fi

echo -e $LOG >> $LOGGING_FILE


#
# Write state to filesystem
#

`rm -f $LRT_FILE`
`rm -f $LO_FILE`

for key in "${!LAST_RESTART_TIMES[@]}"; do
        echo "$key=${LAST_RESTART_TIMES[$key]}" >> "$LRT_FILE"
done

for key in "${!LOCKED_OUT[@]}"; do
        echo "$key=${LOCKED_OUT[$key]}" >> "$LO_FILE"
done