Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public List<DocumentImportEntity> ArangoCollection.importDocuments(
- Collection<?> values,
- DocumentImportOptions options,
- int numThreads,
- int batchSize) {
- // Create a fixed thread pool executorService of numThreads
- ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
- // Use Guava or Apache Commons Collections to partition the List
- // Below, I use Apache Commons Collections to partition the original collection of values into smaller sublists
- List<? extends List<?>> batches = ListUtils.partition(new ArrayList<>(values), batchSize);
- List<CompletableFuture<DocumentImportEntity>> completableFutureList = new ArrayList<>();
- for (List<?> batch : batches) {
- CompletableFuture<DocumentImportEntity> completableFuture = CompletableFuture.supplyAsync(() -> {
- DocumentImportEntity documentImportEntity = importDocuments(batch, documentImportOptions);
- return documentImportEntity;
- }, executorService);
- completableFutureList.add(completableFuture);
- }
- List<DocumentImportEntity> documentImportEntityList = new ArrayList<>();
- for (CompletableFuture<DocumentImportEntity> completableFuture : completableFutureList) {
- DocumentImportEntity documentImportEntity = completableFuture.get();
- documentImportEntityList.add(documentImportEntity);
- }
- return documentImportEntityList;
- }
- DocumentImportOptions documentImportOptions = new DocumentImportOptions()
- .waitForSync(false)
- .onDuplicate(DocumentImportOptions.OnDuplicate.replace)
- .complete(false)
- .details(true);
- Collection<MyPojoObjects> values = collection of 10 million objects;
- int numThreads = 8;
- int batchSize = 1000000; // 1 million
- ArangoCollection.importDocuments(values, documentImportOptions, numThreads, batchSize)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement