daily pastebin goal
31%
SHARE
TWEET

Untitled

a guest Feb 13th, 2018 50 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/bin/bash
  2. #
  3. DIR="$1"        # The source of the millions of files
  4. TARDEST="$2"    # Where the tarballs should be placed
  5.  
  6. # Create the million-file segments
  7. rm -f /tmp/chunk.*
  8. find "$DIR" -type f | split -l 1000000 - /tmp/chunk.
  9.  
  10. # Create corresponding tarballs
  11. for CHUNK in $(cd /tmp && echo chunk.*)
  12. do
  13.     test -f "$CHUNK" || continue
  14.  
  15.     echo "Creating tarball for chunk '$CHUNK'" >&2
  16.     tar cTf "/tmp/$CHUNK" "$TARDEST/$CHUNK.tar"
  17.     rm -f "/tmp/$CHUNK"
  18. done
  19.    
  20. #!/bin/bash
  21. ctr=0;
  22. while readarray -n 1000000 -t asdf; do
  23.   ctr=$((${ctr}+1));
  24.   tar czf /destination/path/asdf.${ctr}.tgz "${asdf[@]}";
  25.   # If you don't want compression, use this instead:
  26.   #tar cf /destination/path/asdf.${ctr}.tar "${asdf[@]}";
  27. done <(find /source/path -not -type d);
  28.    
  29. function something() {...}
  30. find /source/path -not -type d
  31.   | readarray -n 1000000 -t -C something asdf
  32.    
  33. find /source/path -not -type d -print0
  34.   | parallel -j4 -d '' -N1000000 tar czf '/destination/path/thing_backup.{#}.tgz'
  35.    
  36. find /source/path -not -type d -print0
  37.   | parallel -j4 -d '' -N1000000 --tmpdir /destination/path --files tar cz
  38.    
  39. find /source/path -not -type d -print0
  40.   | xargs -P 4 -0 -L 1000000 bash -euc 'tar czf $(mktemp --suffix=".tgz" /destination/path/backup_XXX) "$@"'
  41.    
  42. tar c /source/path | split -b $((3*1024*1024*1024)) - /destination/path/thing.tar.
  43.    
  44. cat $(ls -1 /destination/path/thing.tar.* | sort) | tar x
RAW Paste Data
Top