#!/bin/bash
#==============================================================================
# Name:     snapshot-all-lvs
# Purpose:  Perform daily snapshot management pre-backups
#
# Version:  $Header: $
#==============================================================================

LC_CTYPE=en_US.UTF-8
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/opt/dell/srvadmin/bin:/opt/dell/srvadmin/sbin


#
# Set global variables
#

PROG=$(basename $0)
CONFIG=/opt/ag/etc/${PROG}.cfg
BACKUP_PATH=/srv/backup
LVS_OVERRIDES=/opt/ag/etc/${PROG}.overrides
LOG=$(date +"/tmp/snapshot-all-lvs.%Y%m%d%H%M")

MAILUSER=example@example.com

ERRORS=0

DATE=`date +%d%m%y`
CLEANDATE=`date +%d%m%y --date='2 days ago'`
UMOUNTDATE=`date +%d%m%y --date='1 days ago'`

    # For testing.
#    CLEANDATE=010101
#    UMOUNTDATE=010101
#    DATE=020101
BACKUP_MOUNT_DIR=$BACKUP_PATH/$DATE
export DATE EMAIL BACKUP_PATH lv lv_lvmsize lv_snapsize lvfile LVS_OVERRIDES CONFIG LOG


#
#    Display message with date and time for logging purposes
#
function log_echo () {
    echo -e "`date '+%d-%b-%y %H:%M:%S'` ${PROG}: ${*}" |  tee -a ${LOG}
}


#
# Obtain config overrides - if none set, return default
# Usage: <variable>=$(get_config_override <Override_string> <Default_Value>)
#
function get_config_override () {

    DfltVal=$2
    tmp=$(grep ^$1 $CONFIG | awk -F = '{print $2}')
    if [ -z "$tmp" ] ; then
        echo $DfltVal
    else
        echo $tmp
    fi

}


#
# Obtain LVS overrides - if none set, return default
# Usage: <variable>=$(get_lvs_override <Override_string>)
#
function get_lvs_override () {

    tmp=$(grep ^$1 $LVS_OVERRIDES | awk '{print $2}')
    if [ -z "$tmp" ] ; then
        echo $SNAP_PERCENT
    else
        echo $tmp
    fi

}


function mount_fs () {
    MOUNTDEVICE=$1
    MOUNTPATH=$BACKUP_MOUNT_DIR/$2
    TYPE=$3
    SUCCESS=FALSE
    TYPEOPTION=""
    FSCK_CMD=fsck

    if fsck -n $MOUNTDEVICE 2>&1 |grep -q fsck.drbd
    then
        # we think we're DRBD, but we are really ext3.
        TYPEOPTION="-t ext3"
        FSCK_CMD="fsck.ext3"
    fi
    mkdir -p $MOUNTPATH

    if [ "$TYPE" != "NTFS" ]
    then
        $FSCK_CMD -y $MOUNTDEVICE >& /tmp/snaperrorlogtmp
        if [ $? != 0 ]
        then
            ERRORS=`expr $ERRORS + 1`
            log_echo FSCK $MOUNTDEVICE failed - log follows
            cat /tmp/snaperrorlogtmp >> $LOG
        fi
    fi

    mount $TYPEOPTION -o ro $MOUNTDEVICE $MOUNTPATH >& /tmp/snaperrorlogtmp
    if [ $? != 0 ]
    then
        ERRORS=`expr $ERRORS + 1`
        log_echo mounting /$MOUNTDEVICE failed - log follows
        cat /tmp/snaperrorlogtmp >> $LOG
    fi
    # Confirm successful mount
    if grep -q $MOUNTPATH /proc/mounts
    then
        # successfully mounted
        SUCCESS="TRUE"
        MOUNTCOUNT=`expr $MOUNTCOUNT + 1`
    else
        ERRORS=`expr $ERRORS + 1`
        log_echo "Failure to mount DEVICE $MOUNTDEVICE for $DATE"
    fi
}




function process {
    # Recursive process to recognise LVM, partitions, and filesystems.
    local NAME="$1"
    local DEVICE="$2"
    if [ ! -e $DEVICE  ]
    then
        ERRORS=`expr $ERRORS + 1`
        log_echo DEVICE $DEVICE does not exist.
        return
    fi
    local TYPE=$(file -Lsb $DEVICE 2> /dev/null)
    # Mount filesystems
    if echo $TYPE | grep -qE 'filesystem|Reiser'
    then
        mount_fs $DEVICE $NAME filesystem
        return
    fi
    if echo $TYPE | grep -q NTFS
    then
        mount_fs $DEVICE $NAME NTFS
        return
    fi
    if echo $TYPE | grep -q 'swap file'
    then
        # Swap files cannot be mounted
        return
    fi
    # Import LVM volume groups.
    if [ "$(echo $TYPE|awk '{ print $1}')" = LVM2 ]
    then
        VG="$NAME-$DATE"
        ORIGVG=$(pvs --noheadings -o vg_name $DEVICE|awk '{print $1}')
        vgimportclone -n $VG $DEVICE &> /dev/null
        sleep 2
         # cleanup the leftover LVs - these shouldn't exist.
        for lvname in $(lvs --noheadings -o lv_name $VG)
        do
            dmsetup remove $(echo $ORIGVG|sed s/-/--/g)-$(echo $lvname|sed s/-/--/g)
            lvdevice=$(echo $VG|sed s/-/--/g)-$(echo $lvname|sed s/-/--/g)
            sleep 2 # Queisce
            process $NAME-LVM-$lvname /dev/mapper/$lvdevice
        done
        return
    fi
    # split partitions out and process them (last due to delay)
    DEVMAP=`kpartx -av $DEVICE | awk '{print $3}'`
    PARTITIONS=`echo $DEVMAP | wc -w`
    FILENUM=0
    if [ $PARTITIONS -gt 0 ]
    then
        sleep 3
        for FILE in $DEVMAP
        do
            FILENUM=$(expr $FILENUM + 1)
            process "$NAME-$FILENUM" "/dev/mapper/$FILE"
        done
        return
    fi
    # we still don't know what it is - let's try and mount it anyway
    mount_fs $DEVICE $NAME ''
    if [ $SUCCESS != "TRUE" ]
    then
        print "Unknown volume: $DEVICE $NAME $TYPE"
    fi
}



function cleanup {

    # unmount everything for the UMOUNTDATE.
    unmount $UMOUNTDATE;


    # Decomission VGs
    for VG in $(vgs --noheadings -o vg_name 2> /dev/null |grep $CLEANDATE)
    do
        vgremove --force $VG &> /dev/null
    done

    # remove LVM PVs
    for DEVICE in $(pvs --noheadings -o pv_name 2> /dev/null |grep $CLEANDATE)
    do
        pvremove --force $DEVICE &> /dev/null
        sleep 1
        dmsetup remove $DEVICE &> /dev/null
    done

    # Remove partition mappings
    for DEVICE in $(ls /dev/mapper/*--snapshot--${CLEANDATE}* 2> /dev/null | grep -v cow | grep -v -- "--${CLEANDATE}\$" | sort -r )
    do
        kpartx -d $DEVICE &> /dev/null
        dmsetup remove "$DEVICE" &> /dev/null
        if [ $? -ne 0 ] ; then
            log_echo "Both kpartx -d and dmsetup remove falied: $DEVICE"
        fi
    done


    # remove the actual snapshot.
    for LV in $(lvs  --noheadings -o lv_name,vg_name 2> /dev/null | grep snapshot-${CLEANDATE} | awk '{print $2 "/" $1}')
    do
        lvremove --force $LV &> /dev/null
        if [ $? -ne 0 ];then
            log_echo "Falied to remove snapshot: $LV"
        fi
    done

}



function unmount {
    local MYUNMOUNTDATE=$1
    for FS in $(mount|grep $MYUNMOUNTDATE|awk '{print $3}')
    do
        umount $FS
        if [ $? -ne 0 ] ; then
            log_echo "Failed to umount snapshot $FS"
        else
            rmdir $FS
        fi
    done
    rmdir $BACKUP_PATH/$MYUNMOUNTDATE &> /dev/null
}


#### Main section

# Default snapshot percentage
SNAP_PERCENT="$(get_config_override SNAP_PERCENT 20)"
IGNORE_LVS="$(get_config_override IGNORE_LVS 'swap|temp')"


log_echo "Running for $DATE, cleaning up $CLEANDATE\n\n"

cleanup

printf "Default Snap Percentage: %s\n" $SNAP_PERCENT
printf "LV's to ignore: %s\n" "$IGNORE_LVS"

if [ -s "$LVS_OVERRIDES" ] ; then
{
    printf "`date`: Snap Percentage Overrides:\n\n"
    printf "LV                   Percentage\n"
    printf "==================== ==========\n"
    cat $LVS_OVERRIDES | awk '{ printf "%-20s    %3d\n", $1, $2 }'
    print
} | tee -a $LOG
fi


# Display headings
log_echo " STATUS   Volume Group                   Logical Volume                                SNAP Percent   Mount Count"
log_echo "======== ============================== ============================================= ============== ============="


#### Iterate over logical volumes. - exclude those in the list, exclude those with a dated volume group, and snapshots.
LVS=$(lvs --noheadings -o lv_name,vg_name,origin 2>/dev/null | grep -vE $IGNORE_LVS |grep -vE '[0-9]{6}' | awk '{ if ($3=="") {printf("%s ", $1)}}')
for lv in  $LVS
do
    lv_line=`lvs --noheadings -o lv_name,lv_size,vg_name,origin --units m $VOLUMEGROUP 2>/dev/null | grep " $lv "`
    lv_lvmsize=`echo "$lv_line " \
            | awk '{
                if ($4=="") {\
                    DENOM=substr($2,length($2),1);
                    printf "%d",$2 \
                    } \
                }'`
    if [ -z "$lv_lvmsize" ]
    then
        log_echo ERROR: Weird Issue with LVM with $lv - can\'t calculate snapshot size
        continue
    fi

    # Obtain override, if not set, receives default percentage
    mysnappercent=$(get_lvs_override "$lv ")

    snap_mb=$(expr $(expr $lv_lvmsize \* $mysnappercent) / 100 )
    lv_snapsize=${snap_mb}m
        vg_name=`echo "$lv_line " \
            | awk '{
                if ($4=="") {\
                    printf "%s",$3 \
                    } \
                }'`
    SNAPSHOT="$lv-backup-snapshot-$DATE"

    SNAPSHOT_DEV="/dev/mapper/$vg_name-"$(echo $SNAPSHOT|sed s/-/--/g)

    lvcreate -s -n $SNAPSHOT -L $lv_snapsize $vg_name/$lv >& /tmp/snaperrorlogtmp
    if [ $? != 0 ]
    then
        ERRORS=`expr $ERRORS + 1`
        log_echo "Failed to create snapshot  $NEW_SNAPSHOT  of $vg_name/$lv"
        cat /tmp/snaperrorlogtmp >> $LOG
    fi


    # Create mapping and mount
    MOUNTCOUNT=0
    process $lv $SNAPSHOT_DEV
    if [ $MOUNTCOUNT -eq 0 ]
    then
        fstatus=$(printf "FAILED   %-30s %-45s      %-3.3d         %-2d"  $vg_name  $lv  $mysnappercent  $MOUNTCOUNT)
    else
        fstatus=$(printf "SUCCESS  %-30s %-45s      %-3.3d         %-2d"  $vg_name  $lv  $mysnappercent  $MOUNTCOUNT)
    fi
    log_echo "$fstatus"

done

#
# If not running on a terminal (i.e. cron), cat log file so it appears on the email from cron
#
if [ ! -t 1 ]; then
    echo "\n\n=========== $LOG ===========\n\n"
    cat $LOG
    echo "\n\n========= END $LOG =========\n\n"
fi

#
# If old snapshots exist, email this result
#
OLDSNAPS=`lvs 2>/dev/null | grep backup-snapshot | grep -vE -- "-$DATE|-$UMOUNTDATE"`
if [ "$OLDSNAPS" ] ; then
    (
        printf "************************************************************\n"
        printf "************************************************************\n"
        printf "****** Old Snapshots exist and require manual cleanup ******\n\n"
        lvs | grep backup-snapshot | grep -vE -- "-$DATE|-$UMOUNTDATE"
        printf "\n************************************************************\n"
        printf "************************************************************\n"
    ) | mail -s "`hostname`: Old snapshots exist and require manual cleanup" $MAILUSER
fi