daily pastebin goal
21%
SHARE
TWEET

Untitled

a guest Mar 17th, 2018 60 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/bin/bash
  2.  
  3. # Author: Benjamin MALYNOVYTCH <bmalynovytch#admantic.fr>
  4. # Requires:
  5. #   - aws-cli
  6. #   - pv (for progress)
  7. # Usage: s3-large-file-restore.sh </path/to/destination/file>
  8.  
  9. set -o pipefail
  10. set -e
  11.  
  12. export LC_ALL=en_US.UTF8
  13. export PATH="/usr/local/bin:$PATH"
  14.  
  15. THREADS=8
  16. BUCKET=my_bucket
  17. DEST=$1
  18. BS=$(( 1024 * 1024 ))
  19.  
  20. function restore_chunk {
  21.   CHUNK=$1
  22.   SIZE=$2
  23.   echo "Restoring chunk $CHUNK"
  24.   aws s3 cp s3://$BUCKET/$(basename $DEST)/$CHUNK - | dd of=$DEST bs=$BS seek=$(( $SKIP_O / $BS )) 2> /dev/null
  25. }
  26.  
  27. ( losetup -a | grep -q "$DEST" && umount $DEST ) || true
  28.  
  29. CHUNKS=$(aws s3 ls s3://$BUCKET/$(basename $DEST)/ | grep -v md4sum | awk '{print $4 " " $3}' | sort -n)
  30. ITER=$(echo "$CHUNKS" | wc -l)
  31.  
  32. (
  33.   trap 'kill $(jobs -p) &> /dev/null' EXIT
  34.   SKIP_O=0
  35.   IFS=$'\n'
  36.   for CHUNK_SIZE in $CHUNKS ; do
  37.     while [ $(jobs -p | wc -l) -ge $THREADS ] ; do
  38.       sleep 1
  39.     done
  40.  
  41.     CHUNK=${CHUNK_SIZE% *}
  42.     SIZE=${CHUNK_SIZE#* }
  43.  
  44.     restore_chunk $CHUNK $SIZE &
  45.     let SKIP_O+=$SIZE
  46.   done
  47.   wait
  48. ) | pv -w80 -ls $ITER > /dev/null
  49.  
  50. echo "All done."
RAW Paste Data
Top