Advertisement
Guest User

Untitled

a guest
Jul 21st, 2019
73
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.83 KB | None | 0 0
  1. public List<DocumentImportEntity> ArangoCollection.importDocuments(
  2. Collection<?> values,
  3. DocumentImportOptions options,
  4. int numThreads,
  5. int batchSize) {
  6.  
  7. // Create a fixed thread pool executorService of numThreads
  8. ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
  9.  
  10. // Use Guava or Apache Commons Collections to partition the List
  11. // Below, I use Apache Commons Collections to partition the original collection of values into smaller sublists
  12. List<? extends List<?>> batches = ListUtils.partition(new ArrayList<>(values), batchSize);
  13.  
  14. List<CompletableFuture<DocumentImportEntity>> completableFutureList = new ArrayList<>();
  15. for (List<?> batch : batches) {
  16. CompletableFuture<DocumentImportEntity> completableFuture = CompletableFuture.supplyAsync(() -> {
  17. DocumentImportEntity documentImportEntity = importDocuments(batch, documentImportOptions);
  18. return documentImportEntity;
  19. }, executorService);
  20. completableFutureList.add(completableFuture);
  21. }
  22. List<DocumentImportEntity> documentImportEntityList = new ArrayList<>();
  23. for (CompletableFuture<DocumentImportEntity> completableFuture : completableFutureList) {
  24. DocumentImportEntity documentImportEntity = completableFuture.get();
  25. documentImportEntityList.add(documentImportEntity);
  26. }
  27. return documentImportEntityList;
  28. }
  29.  
  30. DocumentImportOptions documentImportOptions = new DocumentImportOptions()
  31. .waitForSync(false)
  32. .onDuplicate(DocumentImportOptions.OnDuplicate.replace)
  33. .complete(false)
  34. .details(true);
  35.  
  36. Collection<MyPojoObjects> values = collection of 10 million objects;
  37. int numThreads = 8;
  38. int batchSize = 1000000; // 1 million
  39.  
  40. ArangoCollection.importDocuments(values, documentImportOptions, numThreads, batchSize)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement