daily pastebin goal
20%
SHARE
TWEET

Untitled

a guest Mar 17th, 2018 61 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/bin/bash
  2.  
  3. # Author: Benjamin MALYNOVYTCH <bmalynovytch#admantic.fr>
  4. # Requires:
  5. #   - aws-cli
  6. #   - pv (for progress)
  7. # Usage: s3-large-file-restore.sh </path/to/destination/file>
  8.  
  9. set -o pipefail
  10. set -e
  11.  
  12. export LC_ALL=en_US.UTF8
  13. export PATH="/usr/local/bin:$PATH"
  14.  
  15. THREADS=8
  16. BUCKET=my_bucket
  17. DEST=$1
  18. BS=$(( 1024 * 1024 ))
  19.  
  20. function restore_chunk {
  21.   CHUNK=$1
  22.   SIZE=$2
  23.   echo "Restoring chunk $CHUNK"
  24.   aws s3 cp s3://$BUCKET/$(basename $DEST)/$CHUNK - | dd of=$DEST bs=$BS seek=$(( $SKIP_O / $BS )) 2> /dev/null
  25. }
  26.  
  27. ( losetup -a | grep -q "$DEST" && umount $DEST ) || true
  28.  
  29. CHUNKS=$(aws s3 ls s3://$BUCKET/$(basename $DEST)/ | grep -v md4sum | awk '{print $4 " " $3}' | sort -n)
  30. ITER=$(echo "$CHUNKS" | wc -l)
  31.  
  32. (
  33.   trap 'kill $(jobs -p) &> /dev/null' EXIT
  34.   SKIP_O=0
  35.   IFS=$'\n'
  36.   for CHUNK_SIZE in $CHUNKS ; do
  37.     while [ $(jobs -p | wc -l) -ge $THREADS ] ; do
  38.       sleep 1
  39.     done
  40.  
  41.     CHUNK=${CHUNK_SIZE% *}
  42.     SIZE=${CHUNK_SIZE#* }
  43.  
  44.     restore_chunk $CHUNK $SIZE &
  45.     let SKIP_O+=$SIZE
  46.   done
  47.   wait
  48. ) | pv -w80 -ls $ITER > /dev/null
  49.  
  50. echo "All done."
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top