rpi-clone2

#!/bin/bash
#
# Originally written by Bill Wilson, modified by Peter Agoston
# Original code: https://github.com/billw2/rpi-clone

PGM=`basename $0`

RSYNC_OPTIONS="--force -rltWDEgopt"

# List of extra dirs to create under /mnt.
OPTIONAL_MNT_DIRS="clone mnt sda sdb rpi0 rpi1"

# Where to mount the disk filesystems to be rsynced.
CLONE=/mnt/clone

CLONE_LOG=/var/log/$PGM.log

HOSTNAME=`hostname`

SRC_BOOT_PARTITION_TYPE=`parted /dev/mmcblk0 -ms p | grep "^1" | cut -f 5 -d:`
SRC_ROOT_PARTITION_TYPE=`parted /dev/mmcblk0 -ms p | grep "^2" | cut -f 5 -d:`

if [ `id -u` != 0 ]
then
    echo -e "$PGM needs to be run as root.\n"
    exit 1
fi
echo "Checking necessary programs..."
if [[ `which rsync` ]]; then
        echo "Package 'rsync' is installed."
else
    echo "Package 'rsync' is NOT installed."
    echo "Make sure rsync is installed:"
    echo "    $ apt-get update"
    echo -e "    $ apt-get install rsync\n"
    exit 0
fi
if [[ `which kpartx` ]]; then
    echo "Package 'kpartx' is installed."
else
    echo "Package 'kpartx' is NOT installed."
        echo "Installing package 'kpartx'. Please wait..."
        apt-get -y install kpartx || (echo "Installing package 'kpartx' was unsuccesful. Please install it manually!"; exit 1)
fi
if [[ `which pv` ]]; then
    echo "Package 'pv' is installed."
else
    echo "Package 'pv' is NOT installed."
    echo "Installing package 'pv'. Please wait..."
        apt-get -y install pv || (echo "Installing package 'pv' was unsuccesful. Please install it manually!"; exit 1)
fi


usage()
    {
    echo ""
    echo "usage: $PGM sdN {-f|--force-initialize} {-l|--loop} {-v|--verbose} {-y|--yes}"
    echo "    Example:  $PGM sda"
    echo "    -v - list all files as they are copied."
    echo "    -l - destination is not sdN but an image file (created with a loop device)."
    echo "    -f - force initialize the destination partitions."
    echo "    -y - assume 'yes' to all questions during backup. (Silent backup)"
    echo ""
    echo "    Clone (rsync) a running Raspberry Pi file system to a destination"
    echo "    SD card 'sdN' plugged into a Pi USB port (via a USB card reader)."
    echo "    $PGM can clone the running system to a new SD card or can"
    echo "    incrementally rsync to existing backup Raspberry Pi SD cards."
    echo ""
    echo "    If the destination SD card has an existing $SRC_BOOT_PARTITION_TYPE partition 1 and a"
    echo "    $SRC_ROOT_PARTITION_TYPE partition 2, $PGM assumes (unless using the -f option)"
    echo "    that the SD card is an existing backup with the partitions"
    echo "    properly sized and set up for a Raspberry Pi.  All that is needed"
    echo "    is to mount the partitions and rsync them to the running system."
    echo ""
    echo "    If these partitions are not found (or -f), then $PGM will ask"
    echo "    if it is OK to initialize the destination SD card partitions."
    echo "    This is done by a partial 'dd' from the running booted device"
    echo "    /dev/mmcblk0 to the destination SD card /dev/sdN followed by a"
    echo "    fdisk resize and mkfs.ext4 of /dev/sdN partition 2."
    echo "    This creates a completed $SRC_BOOT_PARTITION_TYPE partition 1 containing all boot"
    echo "    files and an empty but properly sized partition 2 rootfs."
    echo "    The SD card  partitions are then mounted and rsynced to the"
    echo "    running system."
    echo ""
    echo "    The SD card destination partitions will be mounted on $CLONE."
    echo "    A log will be written to $CLONE_LOG."
    echo "    Avoid running other disk writing programs when running $PGM."
    echo ""
    echo "    In case of cloning to a local image file (with -l), the filesystem"
    echo "    needs to have a little more than twice as much free space as the"
    echo "    total used space on the source Raspberry Pi, because it keeps"
    echo "    the created temporary image for further (incremental) backups."
    exit 0
    }

VERBOSE=off

while [ "$1" ]
do
    case "$1" in
        -v|--verbose)
            VERBOSE=on
            RSYNC_OPTIONS=${RSYNC_OPTIONS}v
            ;;
        -f|--force-initialize)
            FORCE_INITIALIZE=true
            ;;
        -y|--yes)
            YES=true
            ;;
        -l|--loop)
            LOOP="-o loop"
            DST_FILE=$DST_DISK
            ;;
        -h|--help)
            usage
            ;;
        *)
            if [ "$DST_DISK" != "" ]
            then
                echo "Bad args"
                usage
            fi
            DST_DISK=$1
            ;;
    esac
    shift
done


if [ "$DST_DISK" = "" ]
then
    usage
    exit 0
fi

if [[ ! `cat /proc/partitions | grep -q $DST_DISK` && $LOOP = "" ]]; then
    echo "Destination disk '$DST_DISK' does not exist."
    echo "Plug the destination SD card into a USB port."
    echo "If it does not show up  as '$DST_DISK', then do a"
    echo -e "'cat /proc/partitions' to see where it might be.\n"
    exit 0
fi

unmount_or_abort()
    {
    echo -n "Do you want to unmount $1? (yes/no): "
        [ $YES ] && resp="yes" || read resp
    if [ "$resp" = "y" ] || [ "$resp" = "yes" ]
    then
        if ! umount $1
        then
            echo "Sorry, $PGM could not unmount $1."
            echo -e "Aborting!\n"
            exit 0
        fi
    else
        echo -e "Aborting!\n"
        exit 0
    fi
    }

if [ "$LOOP" != "" ]; then
    DST_ROOT_PARTITION_SIZE=`df | grep '^/dev/root' | awk '{print $3}'`
    DST_BOOT_PARTITION_SIZE=`df | grep '^/dev/mmcblk0p1' | awk '{print $3}'`
    DST_SIZE=$((DST_BOOT_PARTITION_SIZE+DST_ROOT_PARTITION_SIZE+1024*500))
        echo "Checking necessary disk space..."
        DST_FS=$DST_FILE
        until [[ `df|grep -c "${DST_FS}$"` -gt 0 ]]; do DST_FS=${DST_FS%/*}; done #getting filesystem of destination file
    if [ `df -k | grep $DST_FS | awk '{print $4}'` -le $((DST_SIZE*2)) ]; then
        echo "Not enough free space in $DST_FS for the temporary file and the backup image. I need min. $((DST_SIZE*2/1024))MB. Aborting."
        exit 1
    fi
    if [ ! -e $DST_FILE.tmp ]; then
        echo "Temporary loop device file ($DST_FILE.tmp) does not exist yet, creating it now. Well, it may take a while, but this is must"
        echo "be done once only if you won't delete it after backup..."
        (pv -terpb /dev/zero -s ${DST_SIZE}k | dd of=$DST_FILE.tmp bs=1024 count=$DST_SIZE) || (echo "Creating temp file was unsuccesful."; exit 1)
    fi
    echo "Attaching the file to the first free loop device..."
    if [ `losetup -a | grep -c "$DST_FILE.tmp"` -eq 0 ]; then
        (losetup -f $DST_FILE.tmp) || (echo "Loop device could not be attached."; exit 1)
    else
        echo "The file has been already attached to a loop device."
    fi
    if [ $? -gt 0 ]; then
        echo "Attaching was unsuccesful. Do you have any free loop devices?"
        losetup -a
        exit 1
    fi
    LOOP_DEVICE=`losetup -a | grep "$DST_FILE.tmp" | cut -f 1 -d:`
    echo "Used loop device: $LOOP_DEVICE"
    DST_DISK=${LOOP_DEVICE#/dev/}
    DST_ROOT_PARTITION=/dev/mapper/${DST_DISK}p2
    DST_BOOT_PARTITION=/dev/mapper/${DST_DISK}p1
else
    DST_ROOT_PARTITION=/dev/${DST_DISK}
    DST_BOOT_PARTITION=/dev/${DST_DISK}
fi

# Check that none of the destination partitions are busy (mounted).
#
DST_ROOT_CURMOUNT=`fgrep "$DST_ROOT_PARTITION " /etc/mtab | cut -f 2 -d ' ' `
DST_BOOT_CURMOUNT=`fgrep "$DST_BOOT_PARTITION " /etc/mtab | cut -f 2 -d ' ' `

if [ "$DST_ROOT_CURMOUNT" != "" ] || [ "$DST_BOOT_CURMOUNT" != "" ]
then
    echo "A destination partition is busy (mounted).  Mount status:"
    echo "    $DST_ROOT_PARTITION:  $DST_ROOT_CURMOUNT"
    echo "    $DST_BOOT_PARTITION:  $DST_BOOT_CURMOUNT"
    if [ "$DST_BOOT_CURMOUNT" != "" ]
    then
        unmount_or_abort $DST_BOOT_CURMOUNT
    fi
    if [ "$DST_ROOT_CURMOUNT" != "" ]
    then
        unmount_or_abort $DST_ROOT_CURMOUNT
    fi
fi


TEST_MOUNTED=`fgrep " $CLONE " /etc/mtab | cut -f 1 -d ' ' `
if [ "$TEST_MOUNTED" != "" ]
then
    echo "This script uses $CLONE for mounting filesystems, but"
    echo "$CLONE is already mounted with $TEST_MOUNTED."
    unmount_or_abort $CLONE
fi

if [ ! -d $CLONE ]
then
    MNT_MOUNT=`fgrep " /mnt " /etc/mtab | cut -f 1 -d ' ' `
    if [ "$MNT_MOUNT" = "" ]
    then
        mkdir $CLONE
    else
        echo "$MNT_MOUNT is currently mounted on /mnt."
        unmount_or_abort /mnt
        mkdir $CLONE
    fi
fi


# Borrowed from do_expand_rootfs in raspi-config
expand_rootfs()
    {
    # Get the starting offset of the root partition
    PART_START=$(parted /dev/mmcblk0 -ms unit s p | grep "^2" | cut -f 2 -d:)
    [ "$PART_START" ] || return 1
    # Return value will likely be error for fdisk as it fails to reload the
    # partition table because the root fs is mounted
    fdisk /dev/$DST_DISK > /dev/null <<EOF
p
d
2
n
p
2
$PART_START

p
w
q
EOF
    }


# =========== Disk Setup and Checks ===========
#
# Check that destination partitions are the right type.
#
DST_BOOT_PARTITION_TYPE=`parted /dev/$DST_DISK -ms p \
        | grep "^1" | cut -f 5 -d:`
DST_ROOT_PARTITION_TYPE=`parted /dev/$DST_DISK -ms p \
        | grep "^2" | cut -f 5 -d:`


if [ "$DST_BOOT_PARTITION_TYPE" != "$SRC_BOOT_PARTITION_TYPE" ] || \
   [ "$DST_ROOT_PARTITION_TYPE" != "$SRC_ROOT_PARTITION_TYPE" ] || \
   [ "$FORCE_INITIALIZE" = "true" ]
then
    echo ""
    if [ "$FORCE_INITIALIZE" = "true" ]
    then
        echo "*** Forcing a partition initialization of destination '$DST_DISK' ***"
    fi

    echo "The existing partitions on destination disk '$DST_DISK' are:"
#   fdisk -l /dev/$DST_DISK | grep $DST_DISK
    parted /dev/$DST_DISK unit MB p \
        | sed "/^Model/d ; /^Sector/d"
    if [ "$DST_BOOT_PARTITION_TYPE" != "$SRC_BOOT_PARTITION_TYPE" ]
    then
        echo -e "  ... Cannot find a destination boot file system of type: $SRC_BOOT_PARTITION_TYPE\n"
    fi
    if [ "$DST_ROOT_PARTITION_TYPE" != "$SRC_ROOT_PARTITION_TYPE" ]
    then
        echo -e "  ... Cannot find a destination root file system of type: $SRC_ROOT_PARTITION_TYPE\n"
    fi
    echo "This script can initialize the destination disk with a partition"
    echo "structure copied from the currently booted filesytem and then resize"
    echo "partition 2 (the root filesystem) to use all space on the SD card."
    echo -n "Do you want to initialize the destination /dev/$DST_DISK? (yes/no): "
    [ $YES ] && resp="yes" || read resp
    if [ "$resp" = "y" ] || [ "$resp" = "yes" ]
    then
        # Image onto the destination disk a beginning fragment of the
        # running SD card file structure that spans at least more than
        # the start of partition 2.
        #
        # Calculate the start of partition 2 in MB for the dd.
        PART2_START=$(parted /dev/mmcblk0 -ms unit MB p | grep "^2" \
                | cut -f 2 -d: | sed s/MB// | tr "," "." | cut -f 1 -d.)
        # and add some slop
        DD_COUNT=`expr $PART2_START + 8`

        echo ""
        echo "Imaging the partition structure, copying $DD_COUNT megabytes..."
        dd if=/dev/mmcblk0 of=/dev/$DST_DISK bs=1M count=$DD_COUNT

        # But, though Partion 1 is now imaged, partition 2 is incomplete and
        # maybe the wrong size for the destination SD card.  So fdisk it to
        # make it fill the rest of the disk and mkfs it to clean it out.
        #
        echo "Sizing partition 2 (root partition) to use all SD card space..."
        expand_rootfs
        if [[ $LOOP != "" ]]; then
            #parted /dev/$DST_DISK mkfs y 2 ext2
            #tune2fs -O extents,uninit_bg,dir_index /dev/$DST_DISK
            echo "... and we do the partition mappings for the loop device:"
            kpartx -av /dev/$DST_DISK
        fi
        mkfs.ext4 $DST_ROOT_PARTITION > /dev/null

        echo ""
        echo "/dev/$DST_DISK is initialized and resized.  Its partitions are:"
#       fdisk -l /dev/$DST_DISK | grep $DST_DISK
        parted /dev/$DST_DISK unit MB p \
            | sed "/^Model/d ; /^Sector/d"

        SRC_ROOT_VOL_NAME=`e2label /dev/mmcblk0p2`
        echo ""
        echo "Your booted /dev/mmcblk0p2 rootfs existing label: $SRC_ROOT_VOL_NAME"
        echo -n "You may enter a label for the destination rootfs $DST_ROOT_PARTITION: "
        [ $YES ] && resp="" || read resp
        if [ "$resp" != "" ]
        then
            e2label $DST_ROOT_PARTITION $resp
        fi
    else
        echo -e "Aborting\n"
        exit 0
    fi
fi


# =========== Setup Summary ===========
#
DST_ROOT_VOL_NAME=`e2label $DST_ROOT_PARTITION`

if [ "$DST_ROOT_VOL_NAME" = "" ]
then
    DST_ROOT_VOL_NAME="no label"
fi

echo ""
echo "Clone destination disk   :  $DST_DISK"
echo "Clone destination rootfs :  $DST_ROOT_PARTITION ($DST_ROOT_VOL_NAME) on ${CLONE}"
echo "Clone destination bootfs :  $DST_BOOT_PARTITION on ${CLONE}/boot"
echo "Verbose mode             :  $VERBOSE"

echo "==============================="


# If this is an SD card initialization, can watch progress of the clone
# in another terminal with:  watch df -h
#
echo -n "Final check, is it Ok to proceed with the clone (yes/no)?: "
[ $YES ] && resp="yes" || read resp
if [ "$resp" != "y" ] && [ "$resp" != "yes" ]
then
    echo -e "Aborting the disk clone.\n"
    exit 0
fi

#
# =========== End of Setup  ===========




# Mount destination filesystems.

if [[ ! -e $DST_ROOT_PARTITION || ! -e $DST_BOOT_PARTITION ]]; then
    echo "... and we do the partition mappings for the loop device:"
    kpartx -av /dev/$DST_DISK
fi
echo "=> Mounting $DST_ROOT_PARTITION ($DST_ROOT_VOL_NAME) on $CLONE"
if ! mount $LOOP ${DST_ROOT_PARTITION} $CLONE
then
    echo -e "Mount failure of $DST_ROOT_PARTITION, aborting!\n"
    exit 0
fi

if [ ! -d $CLONE/boot ]
then
    mkdir $CLONE/boot
fi

echo "=> Mounting $DST_BOOT_PARTITION on $CLONE/boot"
if ! mount $LOOP ${DST_BOOT_PARTITION} $CLONE/boot
then
    umount $CLONE
    echo -e "Mount failure of $DST_BOOT_PARTITION, aborting!\n"
    exit 0
fi

echo "==============================="




START_TIME=`date '+%H:%M:%S'`

# Exclude fuse mountpoint .gvfs, various other mount points, and tmpfs
# file systems from the rsync.
#
sync
echo "Starting the filesystem rsync to $DST_DISK"
echo -n "(This may take several minutes)..."
rsync $RSYNC_OPTIONS --delete \
        --exclude '.gvfs' \
        --exclude '/dev' \
        --exclude '/media' \
        --exclude '/mnt' \
        --exclude '/proc' \
        --exclude '/run' \
        --exclude '/sys' \
        --exclude '/tmp' \
        --exclude 'lost\+found' \
    // \
    $CLONE



# Fixup some stuff
#

for i in dev media mnt proc run sys
do
    if [ ! -d $CLONE/$i ]
    then
        mkdir $CLONE/$i
    fi
done

if [ ! -d $CLONE/tmp ]
then
    mkdir $CLONE/tmp
    chmod a+w $CLONE/tmp
fi

# Some extra optional dirs I create under /mnt
for i in $OPTIONAL_MNT_DIRS
do
    if [ ! -d $CLONE/mnt/$i ]
    then
        mkdir $CLONE/mnt/$i
    fi
done

rm -f $CLONE/etc/udev/rules.d/70-persistent-net.rules


DATE=`date '+%F %H:%M'`

echo "$DATE  $HOSTNAME $PGM : clone to $DST_DISK ($DST_ROOT_VOL_NAME)" \
        >> $CLONE_LOG
echo "$DATE  $HOSTNAME $PGM : clone to $DST_DISK ($DST_ROOT_VOL_NAME)" \
        >> $CLONE/$CLONE_LOG


STOP_TIME=`date '+%H:%M:%S'`

echo ""
echo "*** Done with clone to /dev/$DST_DISK ***"
echo "    Started: $START_TIME    Finished: $STOP_TIME"
echo ""

# Pause before unmounting in case I want to inspect the clone results
# or need to custom modify any files on the destination SD clone.
# Eg. modify $CLONE/etc/hostname, $CLONE/etc/network/interfaces, etc
# if I'm cloning into a card to be installed on another Pi.
#
echo -n "Hit Enter when ready to unmount the /dev/$DST_DISK partitions..."
[ $YES ] && resp="yes" || read resp

echo "unmounting $CLONE/boot"
umount $CLONE/boot

echo "unmounting $CLONE"
umount $CLONE

if [[ $LOOP != "" ]]; then
    echo "Creating $DST_FILE..."
    pv -terpb $DST_FILE.tmp > $DST_FILE
    echo "Removing loop device mappings..."
    kpartx -ad /dev/$DST_DISK
    echo "Deattaching /dev/$DST_DISK loop device..."
    losetup -d /dev/$DST_DISK
        echo "Hint: If you want to make further backups later, do not delete the temporary file ($DST_FILE.tmp), you can always use that if it's size is still enough."
fi
echo "==============================="

exit 0