Guest User

Untitled

a guest
Feb 25th, 2018
79
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.12 KB | None | 0 0
  1. def md5(fname):
  2. hash_md5 = hashlib.md5()
  3. with open(fname, "rb") as f:
  4. for chunk in iter(lambda: f.read(4096), b""):
  5. hash_md5.update(chunk)
  6. return hash_md5.hexdigest()
  7.  
  8. def file_as_bytes(file):
  9. with file:
  10. return file.read()
  11.  
  12. print hashlib.md5(file_as_bytes(open(full_path, 'rb'))).hexdigest()
  13.  
  14. import hashlib
  15. [(fname, hashlib.md5(file_as_bytes(open(fname, 'rb'))).digest()) for fname in fnamelst]
  16.  
  17. import hashlib
  18. [(fname, hashlib.sha256(file_as_bytes(open(fname, 'rb'))).digest()) for fname in fnamelst]
  19.  
  20. import hashlib
  21.  
  22. def hash_bytestr_iter(bytesiter, hasher, ashexstr=False):
  23. for block in bytesiter:
  24. hasher.update(block)
  25. return (hasher.hexdigest() if ashexstr else hasher.digest())
  26.  
  27. def file_as_blockiter(afile, blocksize=65536):
  28. with afile:
  29. block = afile.read(blocksize)
  30. while len(block) > 0:
  31. yield block
  32. block = afile.read(blocksize)
  33.  
  34.  
  35. [(fname, hash_bytestr_iter(file_as_blockiter(open(fname, 'rb')), hashlib.md5()))
  36. for fname in fnamelst]
  37.  
  38. [(fname, hash_bytestr_iter(file_as_blockiter(open(fname, 'rb')), hashlib.sha256()))
  39. for fname in fnamelst]
  40.  
  41. $ ./sum_methods.py
  42. crc32_mmap(filename) 0.0241742134094
  43. crc32_read(filename) 0.0219960212708
  44. subprocess.check_output(['cksum', filename]) 0.0553209781647
  45. md5sum_mmap(filename) 0.0286180973053
  46. md5sum_read(filename) 0.0311000347137
  47. subprocess.check_output(['md5sum', filename]) 0.0332629680634
  48. $ time md5sum /tmp/test.data.300k
  49. d3fe3d5d4c2460b5daacc30c6efbc77f /tmp/test.data.300k
  50.  
  51. real 0m0.043s
  52. user 0m0.032s
  53. sys 0m0.010s
  54. $ stat -c '%s' /tmp/test.data.300k
  55. 11890400
  56.  
  57. import hashlib
  58. def md5sum(filename, blocksize=65536):
  59. hash = hashlib.md5()
  60. with open(filename, "rb") as f:
  61. for block in iter(lambda: f.read(blocksize), b""):
  62. hash.update(block)
  63. return hash.hexdigest()
  64.  
  65. import zlib
  66. def adler32sum(filename, blocksize=65536):
  67. checksum = zlib.adler32("")
  68. with open(filename, "rb") as f:
  69. for block in iter(lambda: f.read(blocksize), b""):
  70. checksum = zlib.adler32(block, checksum)
  71. return checksum & 0xffffffff
Add Comment
Please, Sign In to add comment