Guest User

Untitled

a guest
Dec 11th, 2018
88
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.91 KB | None | 0 0
  1. # This is a sample class that you'll probably need
  2. # to do a bit of work on for your own needs, but it
  3. # will hopefully save you a lot of the headaches that we had
  4.  
  5. class AwsMove
  6. FIVE_MB = 5 * 1024 * 1024
  7. MAX_THREADS = 25
  8.  
  9. def move_folder(s3_client, old_key, new_key)
  10. # strip preceding '/'
  11. sanitized_old_key = old_key.sub(/\A\//, "")
  12. if params[:mode] == 'rename'
  13. sanitized_new_key = new_key.sub(/\A\//, "")
  14. else
  15. sanitized_new_key = new_key
  16. end
  17.  
  18. t1 = Time.now
  19. Rails.logger.info "Moving"
  20. moves = []
  21.  
  22. objects = s3_client.list_objects_v2(bucket: bucket_name, prefix: sanitized_old_key, encoding_type: 'url')
  23.  
  24. loop do
  25. objects.contents.each do |object|
  26. from_key = CGI::unescape(object.key)
  27. to_key = from_key.sub(sanitized_old_key, sanitized_new_key)
  28. multipart_copy = object.size > FIVE_MB
  29. moves << [from_key, to_key, multipart_copy]
  30. end
  31. break unless objects.next_page?
  32. objects = objects.next_page
  33. end
  34.  
  35. elapsed_ms = ((Time.now - t1) * 1000).to_i
  36. Rails.logger.info "Identified #{moves.count} AWS objects. Total elapsed = #{elapsed_ms}ms"
  37.  
  38. # move files and subfolders before their parent folder
  39. moves.sort_by! { |move| -move[0].length }
  40. queue = moves.inject(Queue.new, :push)
  41. thread_count = [MAX_THREADS, moves.length].min
  42.  
  43. threads = Array.new(thread_count) do
  44. Thread.new do
  45. begin
  46. while move = queue.pop(true)
  47. from_key, to_key, multipart_copy = move
  48. s3_object(s3_client, from_key).move_to(bucket: bucket_name, key: to_key, multipart_copy: multipart_copy)
  49. end
  50. rescue ThreadError
  51. # caught if the queue is empty, or if we are out of memory to generate a new thread
  52. end
  53. end
  54. end
  55. threads.each(&:join)
  56.  
  57. elapsed_ms = ((Time.now - t1) * 1000).to_i
  58. Rails.logger.info "Moved files. Total elapsed = #{elapsed_ms}ms"
  59.  
  60. end
  61. end
Add Comment
Please, Sign In to add comment