Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # frozen_string_literal: true
- class ChunkedWriter
- DEFAULT_CHECK_INTERVAL = 1_000_000
- DEFAULT_MAX_FILE_SIZE_BYTES = (50 * 1024 * 1024).freeze # 50MB
- def initialize(**opts, &finisher)
- @finisher = finisher
- @check_interval = opts.fetch(:check_interval, DEFAULT_CHECK_INTERVAL)
- @max_file_size_bytes = opts.fetch(:max_file_size_bytes, DEFAULT_MAX_FILE_SIZE_BYTES)
- @rows_written_since_last_check = 0
- end
- def write(bytes)
- current_writer.write(bytes).tap { rotate_files_if_necessary }
- end
- def finish
- current_writer.finish # write the gzip footer
- current_file.rewind # yield the file ready for reading
- finisher.call(current_file)
- current_file.close! # close and delete the file
- end
- private
- attr_reader :finisher, :check_interval, :max_file_size_bytes
- def rotate_files_if_necessary
- @rows_written_since_last_check += 1
- return unless @rows_written_since_last_check > check_interval
- @rows_written_since_last_check = 0
- # flush buffers to the output file, so the size can be measured accurately
- current_writer.flush
- return if current_file.size < max_file_size_bytes
- rotate_files
- end
- def rotate_files
- finish
- @current_file = nil
- @current_writer = nil
- end
- def current_writer
- @current_writer ||= Zlib::GzipWriter.new(current_file)
- end
- def current_file
- @current_file ||= Tempfile.new.tap(&:binmode)
- end
- end
Add Comment
Please, Sign In to add comment