Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- while read row; do
- #echo $row
- file_name=$(echo $row | awk '{print $4}')
- #echo $file_name
- #test if the file is already downloaded
- file -E $file_name &> /dev/null
- LOCAL_FILE_TEST_RESULT=$?
- if [ $LOCAL_FILE_TEST_RESULT -eq 0 ]; then
- #echo "${file_name} exists locally - skipping"
- continue
- fi
- # test if we can download the file
- bucket_path="data/Galaxy-Zoo-SDSS/COS_stamps_large/${file_name}"
- bucket="zooniverse-cold-storage"
- s3_path="s3://${bucket}/${bucket_path}"
- aws s3 cp --quiet $s3_path .
- DOWNLOAD_REQ_RESULT=$?
- if [ $DOWNLOAD_REQ_RESULT -eq 0 ]; then
- echo "downloaded ${file_name}"
- echo $file_name > downloaded.log
- else
- aws s3api restore-object --restore-request '{"Days":5,"GlacierJobParameters":{"Tier":"Bulk"}}' --bucket ${bucket} --key ${bucket_path}
- RESTORE_REQUEST_RESULT=$?
- if [ $RESTORE_REQUEST_RESULT -eq 0 ]; then
- echo "requested a restore of ${file_name}"
- echo $file_name > restore_requests.log
- else
- echo "failed to request restore ${file_name}"
- echo $file_name > failed_restores.log
- fi
- fi
- done <file_list.txt
Add Comment
Please, Sign In to add comment