Guest User

Untitled

a guest
Jun 19th, 2018
82
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.34 KB | None | 0 0
  1. public static void splitEvenly(Iterator<String> source, Path... destinations) throws IOException {
  2.  
  3. int numberOfPartitions = destinations.length;
  4. BufferedWriter[] writers = new BufferedWriter[numberOfPartitions];
  5.  
  6. Iterators.overArray(destinations)
  7. .zipWithIndex()
  8. .forEach(destIndexPair -> {
  9. int writerIndex = destIndexPair.getRight().intValue();
  10. Path currentPath = destIndexPair.getLeft();
  11. writers[writerIndex] = Files.newBufferedWriter(currentPath,
  12. StandardOpenOption.CREATE, StandardOpenOption.APPEND
  13. );
  14. }
  15. );
  16.  
  17. try {
  18. source
  19. .zipWithIndex()
  20. .forEach(indexedLine -> {
  21. int currentPartition = (int) (indexedLine.getRight() % numberOfPartitions);
  22. String currentLine = indexedLine.getLeft();
  23. writers[currentPartition].write(currentLine+"\n");
  24. });
  25. } finally {
  26. Iterators.overArray(writers).forEach(BufferedWriter::close);
  27. }
  28. }
  29.  
  30. public static void splitByPages(Iterator<String> source, String destination, int pageSize) throws IOException {
  31.  
  32. source.zipWithIndex().forEach(
  33. new Iterator.ThrowingConsumer<Pair<String, Long>, IOException>() {
  34. BufferedWriter currentWriter;
  35. long currentPage = 0;
  36.  
  37. @Override
  38. public void accept(Pair<String, Long> intexedLine) throws IOException {
  39. long currentPosition = intexedLine.getRight();
  40. String currentLine = intexedLine.getLeft();
  41. long currentPage = currentPosition / pageSize;
  42. if (currentPosition % pageSize == 0) {
  43. if (currentWriter != null) currentWriter.close();
  44. Path currentPath = Paths.get(destination, "part-" + currentPage + ".txt");
  45. currentWriter = Files.newBufferedWriter(currentPath,
  46. StandardOpenOption.CREATE, StandardOpenOption.APPEND);
  47. }
  48. currentWriter.write(currentLine+"\n");
  49. }
  50. }
  51. );
  52. }
Add Comment
Please, Sign In to add comment