Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- public class QueryRunner {
- public static void run(String jobName, String index, String type, String queryPath, String aggsPath, String resultPath, Client client, SearchType searchType, int size, int timeOut, List<SortBuilder> sorts, boolean fields, String... fieldss)
- {
- double startSearchL = 0;
- long hits = 0;
- double totalBytes = 0;
- double maxBytes = 0;
- XContentBuilder builder = null;
- FileOutputStream fop = null;
- try {
- String query = readFileISO_8859_1(queryPath);
- String aggs = new String();
- if(!aggsPath.isEmpty())
- aggs = readFileISO_8859_1(aggsPath);
- File batchFile = new File(resultPath);
- fop = new FileOutputStream(batchFile);
- if (!batchFile.exists()) {
- batchFile.createNewFile();
- }
- else {
- batchFile.delete();
- }
- startSearchL = System.currentTimeMillis();
- SearchRequestBuilder srb = client.prepareSearch(index).setTypes(type)
- .setQuery(query.getBytes())
- .setSearchType(searchType)
- .setScroll(new TimeValue(timeOut))
- .setSize(size);
- for(SortBuilder sort : sorts) {
- srb.addSort(sort);
- }
- if(!aggs.isEmpty())
- srb.setAggregations(aggs.getBytes());
- else
- if(fields)
- srb.addFields(fieldss);
- SearchResponse scrollResp = srb.execute().actionGet();
- hits = scrollResp.getHits().getTotalHits();
- //System.out.println(String.format("%40s - Hits: %d", jobName, hits));
- if(scrollResp.getAggregations() != null) {
- builder = XContentFactory.jsonBuilder();
- builder.startObject();
- scrollResp.toXContent(builder, ToXContent.EMPTY_PARAMS);
- builder.endObject();
- fop.write(builder.string().getBytes());
- } else {
- String header = new String();
- boolean headerWritten = false;
- int count = 0;
- while (true) {
- //System.out.println("Host: " + scrollResp.remoteAddress().toString());
- //System.out.println("Took (ms): " + scrollResp.getTookInMillis());
- //System.out.println(count++);
- for (SearchHit hit : scrollResp.getHits().getHits()) {
- String line = new String();
- if(!fields)
- line = hit.sourceAsString() + "\r\n";
- else
- {
- for(Entry<String, SearchHitField> e : hit.fields().entrySet()){
- header += "\"" + e.getKey() + "\",";
- }
- if(!headerWritten) {
- header += "\r\n";
- //System.out.println(header);
- fop.write(header.getBytes());
- headerWritten = true;
- }
- for(Entry<String, SearchHitField> e : hit.fields().entrySet()){
- line += "\"" + e.getValue().value().toString() + "\",";
- }
- line += "\r\n";
- }
- if(line.length() > maxBytes)
- maxBytes = line.length();
- totalBytes += line.length();
- byte[] bytes = line.getBytes();
- fop.write(bytes);
- }
- scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
- if (scrollResp.getHits().getHits().length == 0) {
- break;
- }
- }
- }
- } catch(Exception ex) {
- System.out.println("Job Failed: " + jobName);
- ex.printStackTrace();
- }
- finally {
- double endSearchL = System.currentTimeMillis();
- double totalTimeMins = ((endSearchL - startSearchL)/1000/60);
- double totalTimeSecs = ((endSearchL - startSearchL)/1000);
- double avgDocSize = hits > 0 ? (totalBytes/hits/1024) : 0;
- double maxDocSize = (maxBytes/1024);
- System.out.println(String.format("%40s - Total time: %10.5f(mins), %10.5f(secs) - Avg doc size: %10.5f(kb) - Max doc size: %10.5f(kb) - Hits: %10d", jobName, totalTimeMins, totalTimeSecs, avgDocSize, maxDocSize, hits));
- //System.out.println(jobName + " - Total time: " + totalTimeMins + "(mins), " + totalTimeSecs + "(secs)" + " - Avg doc size: " + avgDocSize + "(kb)" + " - Max doc size: " + maxDocSize + "(kb)");
- if(builder != null)
- builder.close();
- if(fop != null) {
- try {
- fop.flush();
- fop.close();
- } catch(Exception ex) {
- ex.printStackTrace();
- }
- }
- }
- }
- public static String readFileISO_8859_1(String path) throws IOException {
- return readFile(path, StandardCharsets.ISO_8859_1);
- }
- public static String readFile(String path, Charset encoding) throws IOException {
- byte[] encoded = Files.readAllBytes(Paths.get(path));
- return new String(encoded, encoding);
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement