find_duplicates.sh

#!/bin/bash

# find_duplicates.sh
# Updated: 2012-02-24
# Version: 1.0
# By: Jayson Kempinger < GlowingApple (at) Gmail (dot) com >
# http://www.kempinger.us
# License: GPLv3 (http://www.gnu.org/licenses/gpl-3.0.txt)

# Isolate duplicate files in a directory by recursively computing the MD5 checksums of every file.
# Usage: find_duplicates.sh [ DIR ]

# Known bugs:
#   - some Finder aliases have the same md5 sum, so are misconstrued as duplicates (I assume the link is stored in the resource fork, but md5 only sums the data fork)
#   - certain formats of webloc files are actually empty files (I assume the link is stored in the resource fork, but md5 only sums the data fork)

VERBOSE=1       # Set to 0 to hide output; set to 1 to show some simple output
CHECKSUM=`which md5`    # set to location of md5 command

# Check that the correct number of arguments are given
if [ $# -eq 1 ]; then
    DIR="${1}"
else
    if [ $# -eq 0 ]; then
        DIR="${PWD}"
    else
        echo "Please specify a single directory, or leave blank to use the current directory."
        exit 1
    fi
fi

OLD_IFS="${IFS}"
IFS=$'\n'

if [ $VERBOSE -eq 1 ]; then echo "Checksumming files in ${DIR}..."; fi
FILES=`find "${DIR}" -type f -exec $CHECKSUM -r '{}' \;`
DUPS=`echo "${FILES}" | awk '{print $1}' | uniq -d`

mkdir -p "${DIR}/duplicates"

if [[ $VERBOSE == 1 && "${DUPS}" != "" ]]; then echo "Moving duplicate files to ${DIR}/duplicates..."; fi
for DUP in "${DUPS}"
do
    I=0
    for FILE in `echo "${FILES}" | grep "${DUP}"`
    do
        # move all files, except for first file, to duplicates folder
        if [ $I -ne 0 ]; then
            # Using -n for move, so if two files with the same name exist, will not overwrite
            FILEPATH=`echo "${FILE}" | cut -f 2- -d " "`
            RESULT=`mv -nv "${FILEPATH}" "${DIR}/duplicates/" | grep -c "not overwritten"`
            # Does another file with the same name exist?  If so, append random number to filename and move file.
            if [ $RESULT -ne 0 ]; then
                FILENAME=`basename "${FILEPATH}"`
                NEW_FILENAME="${FILENAME%.*}-$RANDOM.${FILENAME##*.}"
                # $RANDOM should be sufficiently random for this, but just to be safe, using -n to avoid overwritting any data
                mv -n "${FILEPATH}" "${DIR}/duplicates/${NEW_FILENAME}"
            fi
        fi
        let I=$I+1
    done
done

IFS="${OLD_IFS}"

if [ $VERBOSE -eq 1 ]; then
    COUNT=`ls "${DIR}/duplicates" | wc -l | bc`
    echo ""
    if [ $COUNT -eq 0 ]; then
        rmdir "${DIR}/duplicates"
        echo "No duplicate files were found."
    else
        echo -n "${COUNT} duplicate file"
        if [ $COUNT -gt 1 ]; then
            echo -n "s"
        fi
        echo -n " found and moved to "${DIR}/duplicates"; one copy of "
        if [ $COUNT -gt 1 ]; then
            echo -n "each"
        else
            echo -n "the"
        fi
        echo " duplicate set was left in ${DIR}."
        echo ""
        echo "${DIR}:"
        ls -1 "${DIR}/duplicates/"
    fi
fi

exit 0