Guest User

Untitled

a guest
Jun 23rd, 2018
84
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.39 KB | None | 0 0
  1. # frozen_string_literal: true
  2.  
  3. class ChunkedWriter
  4. DEFAULT_CHECK_INTERVAL = 1_000_000
  5. DEFAULT_MAX_FILE_SIZE_BYTES = (50 * 1024 * 1024).freeze # 50MB
  6.  
  7. def initialize(**opts, &finisher)
  8. @finisher = finisher
  9. @check_interval = opts.fetch(:check_interval, DEFAULT_CHECK_INTERVAL)
  10. @max_file_size_bytes = opts.fetch(:max_file_size_bytes, DEFAULT_MAX_FILE_SIZE_BYTES)
  11. @rows_written_since_last_check = 0
  12. end
  13.  
  14. def write(bytes)
  15. current_writer.write(bytes).tap { rotate_files_if_necessary }
  16. end
  17.  
  18. def finish
  19. current_writer.finish # write the gzip footer
  20. current_file.rewind # yield the file ready for reading
  21. finisher.call(current_file)
  22. current_file.close! # close and delete the file
  23. end
  24.  
  25. private
  26.  
  27. attr_reader :finisher, :check_interval, :max_file_size_bytes
  28.  
  29. def rotate_files_if_necessary
  30. @rows_written_since_last_check += 1
  31. return unless @rows_written_since_last_check > check_interval
  32. @rows_written_since_last_check = 0
  33.  
  34. # flush buffers to the output file, so the size can be measured accurately
  35. current_writer.flush
  36. return if current_file.size < max_file_size_bytes
  37.  
  38. rotate_files
  39. end
  40.  
  41. def rotate_files
  42. finish
  43. @current_file = nil
  44. @current_writer = nil
  45. end
  46.  
  47. def current_writer
  48. @current_writer ||= Zlib::GzipWriter.new(current_file)
  49. end
  50.  
  51. def current_file
  52. @current_file ||= Tempfile.new.tap(&:binmode)
  53. end
  54. end
Add Comment
Please, Sign In to add comment