Guest User

Untitled

a guest
Feb 13th, 2018
95
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.39 KB | None | 0 0
  1. #!/bin/bash
  2. #
  3. DIR="$1" # The source of the millions of files
  4. TARDEST="$2" # Where the tarballs should be placed
  5.  
  6. # Create the million-file segments
  7. rm -f /tmp/chunk.*
  8. find "$DIR" -type f | split -l 1000000 - /tmp/chunk.
  9.  
  10. # Create corresponding tarballs
  11. for CHUNK in $(cd /tmp && echo chunk.*)
  12. do
  13. test -f "$CHUNK" || continue
  14.  
  15. echo "Creating tarball for chunk '$CHUNK'" >&2
  16. tar cTf "/tmp/$CHUNK" "$TARDEST/$CHUNK.tar"
  17. rm -f "/tmp/$CHUNK"
  18. done
  19.  
  20. #!/bin/bash
  21. ctr=0;
  22. while readarray -n 1000000 -t asdf; do
  23. ctr=$((${ctr}+1));
  24. tar czf /destination/path/asdf.${ctr}.tgz "${asdf[@]}";
  25. # If you don't want compression, use this instead:
  26. #tar cf /destination/path/asdf.${ctr}.tar "${asdf[@]}";
  27. done <(find /source/path -not -type d);
  28.  
  29. function something() {...}
  30. find /source/path -not -type d
  31. | readarray -n 1000000 -t -C something asdf
  32.  
  33. find /source/path -not -type d -print0
  34. | parallel -j4 -d '' -N1000000 tar czf '/destination/path/thing_backup.{#}.tgz'
  35.  
  36. find /source/path -not -type d -print0
  37. | parallel -j4 -d '' -N1000000 --tmpdir /destination/path --files tar cz
  38.  
  39. find /source/path -not -type d -print0
  40. | xargs -P 4 -0 -L 1000000 bash -euc 'tar czf $(mktemp --suffix=".tgz" /destination/path/backup_XXX) "$@"'
  41.  
  42. tar c /source/path | split -b $((3*1024*1024*1024)) - /destination/path/thing.tar.
  43.  
  44. cat $(ls -1 /destination/path/thing.tar.* | sort) | tar x
Add Comment
Please, Sign In to add comment