Remove Duplicates

find duplicates in directory

import sys, os, hashlib, resource

if __name__ == '__main__':
	if len(sys.argv) < 2: # error, # args
	dirName = sys.argv[1]
	if not (os.path.exists(dirName) and os.path.isdir(dirName)): # error, not a dir
	print(dupes(dirName))

def getPaths(dirName):
	paths = []
	for dirName, subdirList, fileList in os.walk(dirName):
		for fileName in fileList:
			paths.append(os.path.join(dirName, fileName))
	return paths

def getChecksum(path):
	checksum = hashlib.md5()
	pagesize = resource.getpagesize()
	with open(path, 'r') as f:
		buf = f.read(pagesize)
		if not buf: break
		checksum.update(buf)
	return checksum.digest()

# good-enough unique identifier for a path
def getCompoundKey(path):
	return (os.path.getsize(path), getChecksum(path))

def dupes(dirName):
	paths = getPaths(dirName)
	dupes, records = [], {}
	for path in paths:
		key = getCompoundKey(path)
		if key in records: dupes.append(key)
		else: records.add(key)
	for dup in dupes:
		os.remove(dup)