Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- for (String fromIndex : indexList) {
- SearchResponse rsp = client.prepareSearch(fromIndex).
- setQuery(QueryBuilders.matchAllQuery()).setSize(hitsPerPage).
- // This is important:
- setSearchType(SearchType.QUERY_AND_FETCH).
- setScroll(TimeValue.timeValueMinutes(30)).execute().actionGet();
- try {
- long total = rsp.hits().totalHits();
- Collection<MyTweet> tweets = collectTweets(rsp);
- bulkUpdate(tweets, intoIndex);
- int page = 1;
- int pages = (int) (total / hitsPerPage);
- if (total % hitsPerPage > 0)
- pages++;
- int collectedResults = 0;
- int currentResults;
- while ((currentResults = rsp.hits().hits().length) > 0) {
- collectedResults+=currentResults;
- rsp = client.prepareSearchScroll(rsp.scrollId()).
- setScroll(TimeValue.timeValueMinutes(30)).execute().actionGet();
- tweets = collectTweets(rsp);
- bulkUpdate(tweets, intoIndex);
- logger.info("Progress " + page++ + "/"+ pages + " current=" + currentResults
- + " total=" + total + " fromIndex=" + fromIndex);
- }
- logger.info("Finished copying of index:"+fromIndex+". Total:" + total + " collected:" + collectedResults);
- } catch (Exception ex) {
- logger.error("Failed to copy data from index " + fromIndex + " into " + intoIndex + ".", ex);
- }
- }
Add Comment
Please, Sign In to add comment