::: {server0057-3}{8pk_r9BTQbqnUeSq12KwWQ}{lywzWPkET262tMdN3HcRVQ}{192.168.2.163}{192.168.2.163:9303}{ml.machine_memory=279478022144, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.486Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   120.2% (600.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-3][search][T#36]'
     10/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   117.1% (585.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-3][search][T#26]'
     10/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   116.9% (584.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-3][search][T#25]'
     10/10 snapshots sharing following 35 elements
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0301}{7aj1J40NRv2PmJ7hbi8xOQ}{x1bJYKCRQEmsQ_HHJbskoQ}{192.168.3.44}{192.168.3.44:9301}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.727Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   96.9% (484.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0301][search][T#109]'
     3/10 snapshots sharing following 43 elements
       java.nio.Bits.copyToArray(Bits.java:835)
       java.nio.DirectByteBuffer.get(DirectByteBuffer.java:280)
       org.apache.lucene.store.ByteBufferGuard.getBytes(ByteBufferGuard.java:93)
       org.apache.lucene.store.ByteBufferIndexInput.readBytes(ByteBufferIndexInput.java:89)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:173)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 36 elements
       org.apache.lucene.index.ReaderUtil.getTopLevelContext(ReaderUtil.java:36)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.shouldCache(LRUQueryCache.java:707)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:724)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:428)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   88.0% (439.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0301][search][T#57]'
     5/10 snapshots sharing following 39 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 29 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   80.0% (400.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0301][search][T#48]'
     5/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:751)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0057-4}{p2KSd2N8QsKI9-yaS7UMVQ}{i5GGtOT0RNCqJkYqZpXqgw}{192.168.2.163}{192.168.2.163:9304}{ml.machine_memory=279478022144, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.481Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   85.9% (429.5ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-4][search][T#75]'
     8/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:47)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:438)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 34 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   82.8% (413.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-4][search][T#59]'
     10/10 snapshots sharing following 44 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:98)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:736)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   70.0% (350.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0057-4][search][T#50]'
     10/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getArc(SegmentTermsEnum.java:229)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:483)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:751)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0205-2}{YMerq0x4Q26Wl-5jR_Jm1g}{cUBGnQq5TDiX6-bY6iJYgw}{192.168.2.162}{192.168.2.162:9302}{ml.machine_memory=278922899456, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.456Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   102.4% (511.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-2][search][T#38]'
     4/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 34 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   92.3% (461.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-2][search][T#15]'
     2/10 snapshots sharing following 35 elements
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 41 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.index.TermContext.build(TermContext.java:96)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 16 elements
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0301-5}{WZwUFu7hQNOv_XVcGKnm2A}{rKEMmey3TWOjoqHy-4Yu8A}{192.168.3.44}{192.168.3.44:9305}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:31.405Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   40.2% (201ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-5][search][T#62]'
     8/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   39.4% (197.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-5][search][T#81]'
     10/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   39.4% (197ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-5][search][T#76]'
     10/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getArc(SegmentTermsEnum.java:229)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:483)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0425}{wpMnCP6oR7mZs_6F1buq6Q}{Q8KLPSHqQpiZoAOKid96cA}{192.168.3.196}{192.168.3.196:9300}{ml.machine_memory=101334450176, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=warm}
   Hot threads at 2019-07-20T13:42:30.456Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   26.6% (133.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0425][transport_worker][T#32]'
     2/10 snapshots sharing following 25 elements
       sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:963)
       sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:896)
       sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:766)
       javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624)
       io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295)
       io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1301)
       io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
       io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
       io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
       io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
       io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
       io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
       io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
       io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
       io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
       io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
       io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 27 elements
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
       io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
       io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1436)
       io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
       io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
       io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
       io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
       io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
       io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
       io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
       io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
       io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
       io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
       io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 9 elements
       sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
       sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
       sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
       sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
       sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
       io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765)
       io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413)
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)

   24.3% (121.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0425][transport_worker][T#31]'
     2/10 snapshots sharing following 35 elements
       sun.security.provider.DigestBase.implCompressMultiBlock(DigestBase.java:141)
       sun.security.provider.DigestBase.engineUpdate(DigestBase.java:128)
       java.security.MessageDigestSpi.engineUpdate(MessageDigestSpi.java:114)
       java.security.MessageDigest$Delegate.engineUpdate(MessageDigest.java:588)
       java.security.MessageDigest.update(MessageDigest.java:353)
       com.sun.crypto.provider.HmacCore.engineUpdate(HmacCore.java:182)
       javax.crypto.Mac.update(Mac.java:534)
       sun.security.ssl.MAC.compute(MAC.java:162)
       sun.security.ssl.EngineInputRecord.checkMacTags(EngineInputRecord.java:297)
       sun.security.ssl.EngineInputRecord.decrypt(EngineInputRecord.java:236)
       sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:963)
       sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:896)
       sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:766)
       javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624)
       io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295)
       io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1301)
       io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1203)
       io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1247)
       io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:502)
       io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:441)
       io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:278)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
       io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
       io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
       io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
       io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
       io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656)
       io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556)
       io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510)
       io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470)
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)
     8/10 snapshots sharing following 9 elements
       sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
       sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
       sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
       sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
       sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
       io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:765)
       io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:413)
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)

::: {server0355}{Ssz0IBvSTkq0Lk2Qr5OTFg}{7-0ry5bkR4euWZyLsMIfrA}{192.168.3.128}{192.168.3.128:9301}{ml.machine_memory=84457807872, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=warm}
   Hot threads at 2019-07-20T13:42:30.455Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0172-2}{8bU9avYoSE6xWLrwSpDywg}{nEGGah_9QtWTJSN5hLuUmw}{192.168.3.125}{192.168.3.125:9302}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.455Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   23.1% (115.6ms out of 500ms) cpu usage by thread 'elasticsearch[server0172-2][search][T#73]'
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 34 elements
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 32 elements
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0208}{qwzJE_zXTQeoOJ-Wq5-QaA}{uc01D59_T0-jyGkgpmm8ew}{192.168.2.166}{192.168.2.166:9301}{ml.machine_memory=84630806528, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=warm}
   Hot threads at 2019-07-20T13:42:30.455Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0301-2}{UFP4eqLBR5K86U_oFXBOCA}{FcfwDP1dQ4OPx4QOMV-2pw}{192.168.3.44}{192.168.3.44:9302}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.547Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   86.9% (434.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-2][search][T#79]'
     5/10 snapshots sharing following 32 elements
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 37 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   85.9% (429.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-2][search][T#115]'
     5/10 snapshots sharing following 41 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:47)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:428)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   82.4% (412.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-2][search][T#103]'
     3/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 44 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.terms(BlockTreeTermsReader.java:265)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:307)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:428)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0172-4}{uOZ0QULZTJOwkAO1sw21mA}{2ZrAbUIPRzuwgVEf34yiMg}{192.168.3.125}{192.168.3.125:9304}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.455Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   93.3% (466.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0172-4][search][T#178]'
     4/10 snapshots sharing following 34 elements
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     6/10 snapshots sharing following 43 elements
       java.lang.Class.getEnclosingMethod0(Native Method)
       java.lang.Class.getEnclosingMethodInfo(Class.java:1072)
       java.lang.Class.getEnclosingClass(Class.java:1272)
       java.lang.Class.getSimpleBinaryName(Class.java:1443)
       java.lang.Class.getSimpleName(Class.java:1309)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.isPointQuery(UsageTrackingQueryCachingPolicy.java:39)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.isCostly(UsageTrackingQueryCachingPolicy.java:54)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.minFrequencyToCache(UsageTrackingQueryCachingPolicy.java:121)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.shouldCache(UsageTrackingQueryCachingPolicy.java:179)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:747)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {client}{dU-F_RcPRg63UHBBj3m53Q}{PdNMbW_xTEChFYNrHXXrgw}{192.168.2.236}{192.168.2.236:9301}{ml.machine_memory=25337901056, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}
   Hot threads at 2019-07-20T13:42:30.455Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {master2}{2Z7Q17L3QLexNoIemrFDLg}{ixyfOj1PQRKeQGq0barMvQ}{192.168.3.39}{192.168.3.39:9301}{ml.machine_memory=14748729344, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
   Hot threads at 2019-07-20T13:42:30.456Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {master}{_KTxllpsTb-qAmH2hHOWRw}{LyVAVa4ETXWFSPOhUwAN7w}{192.168.2.165}{192.168.2.165:9301}{ml.machine_memory=23223640064, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
   Hot threads at 2019-07-20T13:42:30.457Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

    0.1% (716.3micros out of 500ms) cpu usage by thread 'elasticsearch[master][transport_worker][T#8]'
     8/10 snapshots sharing following 2 elements
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)

    0.1% (356.1micros out of 500ms) cpu usage by thread 'elasticsearch[master][transport_worker][T#11]'
     9/10 snapshots sharing following 2 elements
       io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909)
       java.lang.Thread.run(Thread.java:748)

::: {server0173-3}{0yiyd_nzRcSRYkO8xqpw9g}{JhDWxb_KT7y3yuNOlzJUSw}{192.168.3.126}{192.168.3.126:9303}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.457Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   94.2% (471.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0173-3][search][T#54]'
     10/10 snapshots sharing following 2 elements
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   19.6% (97.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0173-3][search][T#94]'
     7/10 snapshots sharing following 36 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0204-3}{MIkNxSPFTQy_qSt7weU26Q}{q4Zm2_jMTduqYhZD5vM_jA}{192.168.2.161}{192.168.2.161:9303}{ml.machine_memory=278922874880, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.908Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   86.6% (432.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-3][search][T#68]'
     2/10 snapshots sharing following 42 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:171)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     8/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   85.5% (427.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-3][search][T#13]'
     6/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:50)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 44 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:98)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   83.5% (417.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-3][search][T#32]'
     10/10 snapshots sharing following 34 elements
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0205-3}{wTGW2GrbQwilvPjPNucNBA}{d3odT9UxQzypXe2xDIemrw}{192.168.2.162}{192.168.2.162:9303}{ml.machine_memory=278922899456, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.461Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   84.4% (421.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-3][search][T#15]'
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     8/10 snapshots sharing following 2 elements
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   75.0% (374.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-3][search][T#3]'
     3/10 snapshots sharing following 34 elements
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     7/10 snapshots sharing following 28 elements
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0204-4}{r3WQsY1dSDOyUb5vTUDEIQ}{VMiaJxM7RWaPXdyRVe1YMA}{192.168.2.161}{192.168.2.161:9304}{ml.machine_memory=278922874880, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.464Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   30.5% (152.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-4][search][T#20]'
     4/10 snapshots sharing following 37 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:63)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.index.TermContext.build(TermContext.java:98)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     6/10 snapshots sharing following 41 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.index.TermContext.build(TermContext.java:96)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   30.0% (150.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-4][search][T#34]'
     10/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getArc(SegmentTermsEnum.java:226)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:483)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   29.8% (149ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-4][search][T#18]'
     5/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 32 elements
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0200}{pFt_8oXqSymS0eAaJ6CSpg}{PT_fZ-AGQCmsVlqxp06-Vg}{192.168.2.148}{192.168.2.148:9301}{ml.machine_memory=84627787776, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=warm}
   Hot threads at 2019-07-20T13:42:30.460Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0300-4}{yIci2O-ITIuwOuc2abUpDQ}{Ue2f18ufRuOkNAYJ7fGPVA}{192.168.3.43}{192.168.3.43:9304}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.459Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   90.1% (450.6ms out of 500ms) cpu usage by thread 'elasticsearch[server0300-4][search][T#132]'
     10/10 snapshots sharing following 43 elements
       java.lang.Class.getEnclosingMethod0(Native Method)
       java.lang.Class.getEnclosingMethodInfo(Class.java:1072)
       java.lang.Class.getEnclosingClass(Class.java:1272)
       java.lang.Class.getSimpleBinaryName(Class.java:1443)
       java.lang.Class.getSimpleName(Class.java:1309)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.isPointQuery(UsageTrackingQueryCachingPolicy.java:39)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.isCostly(UsageTrackingQueryCachingPolicy.java:54)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.minFrequencyToCache(UsageTrackingQueryCachingPolicy.java:121)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.shouldCache(UsageTrackingQueryCachingPolicy.java:179)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:747)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0173}{7IJhiC5xTBO--VWLGJcOig}{gUgKCy5XTpSJuqTUClpwHw}{192.168.3.126}{192.168.3.126:9301}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.460Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0057}{Ck4UK6jwSBGUe92P_vjD3A}{GTC2DW-pSwGNoo2hWAMWQA}{192.168.2.163}{192.168.2.163:9301}{ml.machine_memory=279478022144, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.794Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   99.4% (496.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0057][search][T#48]'
     10/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   95.3% (476.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0057][search][T#9]'
     3/10 snapshots sharing following 44 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:51)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     7/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   86.1% (430.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0057][search][T#44]'
     4/10 snapshots sharing following 35 elements
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     6/10 snapshots sharing following 44 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:98)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0173-4}{YoS6gRPER0KJzC6eGmV32A}{B72SFU8qRD2NwAaYH_fI2Q}{192.168.3.126}{192.168.3.126:9304}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.460Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

    1.7% (8.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0173-4][management][T#1]'
     6/10 snapshots sharing following 2 elements
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0301-4}{kJb4TWh8RGGeYLL3C_QDAA}{d1Ckg2iKSKuePX_XYfXrFw}{192.168.3.44}{192.168.3.44:9304}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.462Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   93.0% (464.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-4][search][T#74]'
     2/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:50)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getArc(SegmentTermsEnum.java:226)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:483)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 36 elements
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 31 elements
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   92.1% (460.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-4][search][T#77]'
     5/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 35 elements
       org.apache.lucene.index.TermContext.build(TermContext.java:98)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 35 elements
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   88.2% (440.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-4][search][T#62]'
     2/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 37 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 41 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:98)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0205}{lgJadzSsQlmoH3RaJC7PeA}{BrZ_C0XuTsubdEgnYFPRxg}{192.168.2.162}{192.168.2.162:9301}{ml.machine_memory=278922899456, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.463Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   101.1% (505.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0205][search][T#37]'
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     7/10 snapshots sharing following 10 elements
       sun.misc.Unsafe.park(Native Method)
       java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
       java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
       java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
       java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
       org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
       java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   91.4% (457.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0205][search][T#4]'
     4/10 snapshots sharing following 41 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.index.TermContext.build(TermContext.java:96)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 34 elements
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       java.util.Arrays.hashCode(Arrays.java:4146)
       org.apache.lucene.search.DisjunctionMaxQuery.hashCode(DisjunctionMaxQuery.java:279)
       java.util.AbstractSet.hashCode(AbstractSet.java:126)
       java.util.EnumMap.entryHashCode(EnumMap.java:717)
       java.util.EnumMap.hashCode(EnumMap.java:709)
       java.util.Arrays.hashCode(Arrays.java:4146)
       java.util.Objects.hash(Objects.java:128)
       org.apache.lucene.search.BooleanQuery.computeHashCode(BooleanQuery.java:518)
       org.apache.lucene.search.BooleanQuery.hashCode(BooleanQuery.java:532)
       org.apache.lucene.search.UsageTrackingQueryCachingPolicy.onUse(UsageTrackingQueryCachingPolicy.java:150)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:714)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0205-4}{96zg_3K8T-CRgFfr-LtlPA}{nLuzqRAORnCMQ6wxySEAow}{192.168.2.162}{192.168.2.162:9304}{ml.machine_memory=278922899456, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.463Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   103.2% (515.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-4][search][T#96]'
     2/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     5/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 31 elements
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   94.8% (474ms out of 500ms) cpu usage by thread 'elasticsearch[server0205-4][search][T#39]'
     5/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 37 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.index.TermContext.build(TermContext.java:98)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 34 elements
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0301-3}{PslOCpoWST20epjVkpxqOA}{qZni0FBZSjmS2zEgohZRTA}{192.168.3.44}{192.168.3.44:9303}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.463Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   94.4% (472.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-3][search][T#13]'
     2/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 37 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 40 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.terms(BlockTreeTermsReader.java:265)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:307)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   85.1% (425.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-3][search][T#15]'
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 37 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 34 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 43 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:428)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:438)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   77.7% (388.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0301-3][search][T#21]'
     7/10 snapshots sharing following 41 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.index.TermContext.build(TermContext.java:96)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:212)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       java.nio.Bits.copyToArray(Bits.java:835)
       java.nio.DirectByteBuffer.get(DirectByteBuffer.java:280)
       org.apache.lucene.store.ByteBufferGuard.getBytes(ByteBufferGuard.java:93)
       org.apache.lucene.store.ByteBufferIndexInput.readBytes(ByteBufferIndexInput.java:89)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.loadBlock(SegmentTermsEnumFrame.java:173)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0171-4}{G4QlrSLuTFWg1eamr1mWFg}{Xzb65iY1Tkmf4WNV6PDqMQ}{192.168.2.104}{192.168.2.104:9304}{data=hot, xpack.installed=true}
   Hot threads at 2019-07-20T13:42:30.464Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   106.3% (531.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0171-4][search][T#55]'
     2/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 42 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 29 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   106.3% (531.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0171-4][search][T#77]'
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 42 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 29 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       java.lang.Object.hashCode(Native Method)
       org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$Key.hashCode(IndicesFieldDataCache.java:244)
       org.elasticsearch.common.cache.Cache.getCacheSegment(Cache.java:859)
       org.elasticsearch.common.cache.Cache.get(Cache.java:364)
       org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:391)
       org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:169)
       org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
       org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsMapping(ValuesSource.java:158)
       org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator$LowCardinality.getLeafCollector(GlobalOrdinalsStringTermsAggregator.java:338)
       org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:167)
       org.elasticsearch.search.aggregations.AggregatorBase.getLeafCollector(AggregatorBase.java:40)
       org.apache.lucene.search.MultiCollector.getLeafCollector(MultiCollector.java:121)
       org.apache.lucene.search.FilterCollector.getLeafCollector(FilterCollector.java:40)
       org.elasticsearch.search.query.CancellableCollector.getLeafCollector(CancellableCollector.java:51)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:661)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   105.9% (529.4ms out of 500ms) cpu usage by thread 'elasticsearch[server0171-4][search][T#80]'
     3/10 snapshots sharing following 37 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 34 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:717)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 39 elements
       org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.postings(Lucene50PostingsReader.java:220)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1002)
       org.apache.lucene.search.PhraseQuery$1.getPhraseMatcher(PhraseQuery.java:449)
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 35 elements
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       org.apache.lucene.util.PriorityQueue.<init>(PriorityQueue.java:100)
       org.apache.lucene.util.PriorityQueue.<init>(PriorityQueue.java:47)
       org.apache.lucene.search.DisjunctionScorer$TwoPhase$1.<init>(DisjunctionScorer.java:98)
       org.apache.lucene.search.DisjunctionScorer$TwoPhase.<init>(DisjunctionScorer.java:98)
       org.apache.lucene.search.DisjunctionScorer$TwoPhase.<init>(DisjunctionScorer.java:87)
       org.apache.lucene.search.DisjunctionScorer.<init>(DisjunctionScorer.java:69)
       org.apache.lucene.search.DisjunctionMaxScorer.<init>(DisjunctionMaxScorer.java:44)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:148)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0201}{A8zoiN_CQke2uMDUhqCZKg}{-GKM4lICTcWVv8BxVbFU1w}{192.168.2.150}{192.168.2.150:9301}{ml.machine_memory=101575426048, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=warm}
   Hot threads at 2019-07-20T13:42:30.461Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0204-2}{PK5jfYyERIGUQ8GRuaNWdQ}{58dGphXJRvyEfZm7vHBCvg}{192.168.2.161}{192.168.2.161:9302}{ml.machine_memory=278922874880, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.472Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   79.6% (397.8ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-2][search][T#96]'
     10/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:507)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   77.2% (386.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-2][search][T#83]'
     10/10 snapshots sharing following 31 elements
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   76.2% (381.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0204-2][search][T#71]'
     9/10 snapshots sharing following 35 elements
       org.apache.lucene.index.TermContext.build(TermContext.java:99)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.index.TermContext.build(TermContext.java:96)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0172}{NmXQyPmITD68SGR5HVDcBw}{51qmcv5XSZCMTXg8lh6-kQ}{192.168.3.125}{192.168.3.125:9301}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.461Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0300}{BfioOhYvT0yOs1i7WuOa0Q}{Ob0ihfY2RXygclpuF6jxYg}{192.168.3.43}{192.168.3.43:9301}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.461Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   19.2% (95.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0300][search][T#131]'
     4/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 37 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:741)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 40 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 29 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0172-3}{HYDjKTm2SL65ju-gXzEbuQ}{VS8Z05vXSSW1XjOKbQL_8g}{192.168.3.125}{192.168.3.125:9303}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.478Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   98.2% (490.9ms out of 500ms) cpu usage by thread 'elasticsearch[server0172-3][search][T#135]'
     6/10 snapshots sharing following 36 elements
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0300-2}{A9YdQ4YzRCODJ83vWcWeQA}{l2vfBI92RzuinHKBr4q9UA}{192.168.3.43}{192.168.3.43:9302}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.462Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   97.0% (485ms out of 500ms) cpu usage by thread 'elasticsearch[server0300-2][search][T#45]'
     2/10 snapshots sharing following 37 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:144)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.PhraseWeight.scorer(PhraseWeight.java:46)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     2/10 snapshots sharing following 40 elements
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0171}{uWSVTEDNT5yhLbyaAehnMA}{AHTgp62rR2eU50zkj7cofQ}{192.168.2.104}{192.168.2.104:9301}{data=hot, xpack.installed=true}
   Hot threads at 2019-07-20T13:42:30.476Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0204}{l-4atYiSTVGh0Rp8wDvluQ}{eg-ADWdETvS_wVDfd64DHg}{192.168.2.161}{192.168.2.161:9301}{ml.machine_memory=278922874880, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.971Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   81.9% (409.3ms out of 500ms) cpu usage by thread 'elasticsearch[server0204][refresh][T#5]'
     10/10 snapshots sharing following 33 elements
       org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:577)
       org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
       org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:221)
       org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:251)
       org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:135)
       org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:469)
       org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
       org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
       org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:493)
       org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:294)
       org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:269)
       org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:259)
       org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
       org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
       org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
       org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
       org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
       org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
       org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:326)
       org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:301)
       org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
       org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
       org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1639)
       org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1620)
       org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:967)
       org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:746)
       org.elasticsearch.index.IndexService.access$500(IndexService.java:100)
       org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:875)
       org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:141)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   81.4% (407.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0204][search][T#49]'
     10/10 snapshots sharing following 38 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnumFrame.<init>(SegmentTermsEnumFrame.java:53)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:77)
       org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:156)
       org.apache.lucene.index.TermContext.build(TermContext.java:98)
       org.apache.lucene.search.PhraseQuery$1.getStats(PhraseQuery.java:407)
       org.apache.lucene.search.PhraseWeight.<init>(PhraseWeight.java:37)
       org.apache.lucene.search.PhraseQuery$1.<init>(PhraseQuery.java:389)
       org.apache.lucene.search.PhraseQuery.createWeight(PhraseQuery.java:389)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.<init>(DisjunctionMaxQuery.java:106)
       org.apache.lucene.search.DisjunctionMaxQuery.createWeight(DisjunctionMaxQuery.java:196)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.BooleanWeight.<init>(BooleanWeight.java:54)
       org.apache.lucene.search.BooleanQuery.createWeight(BooleanQuery.java:207)
       org.apache.lucene.search.IndexSearcher.createWeight(IndexSearcher.java:751)
       org.elasticsearch.search.internal.ContextIndexSearcher.createWeight(ContextIndexSearcher.java:148)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   76.8% (384.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0204][search][T#95]'
     4/10 snapshots sharing following 28 elements
       org.elasticsearch.index.mapper.NumberFieldMapper$NumberFieldType.termQuery(NumberFieldMapper.java:923)
       org.elasticsearch.index.search.QueryParserHelper.resolveMappingField(QueryParserHelper.java:149)
       org.elasticsearch.index.search.QueryParserHelper.resolveMappingField(QueryParserHelper.java:110)
       org.elasticsearch.index.search.QueryStringQueryParser.<init>(QueryStringQueryParser.java:142)
       org.elasticsearch.index.query.QueryStringQueryBuilder.doToQuery(QueryStringQueryBuilder.java:1000)
       org.elasticsearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:105)
       org.elasticsearch.index.query.BoolQueryBuilder.addBooleanClauses(BoolQueryBuilder.java:415)
       org.elasticsearch.index.query.BoolQueryBuilder.doToQuery(BoolQueryBuilder.java:383)
       org.elasticsearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:105)
       org.elasticsearch.index.query.QueryShardContext.lambda$toQuery$2(QueryShardContext.java:319)
       org.elasticsearch.index.query.QueryShardContext$$Lambda$3682/369120218.apply(Unknown Source)
       org.elasticsearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:331)
       org.elasticsearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:318)
       org.elasticsearch.search.SearchService.parseSource(SearchService.java:791)
       org.elasticsearch.search.SearchService.createContext(SearchService.java:637)
       org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:596)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:387)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     6/10 snapshots sharing following 34 elements
       org.elasticsearch.common.collect.CopyOnWriteHashMap$InnerNode.visit(CopyOnWriteHashMap.java:278)
       org.elasticsearch.common.collect.CopyOnWriteHashMap$EntryIterator.next(CopyOnWriteHashMap.java:454)
       org.elasticsearch.common.collect.CopyOnWriteHashMap$EntryIterator.next(CopyOnWriteHashMap.java:431)
       java.util.AbstractMap$2$1.next(AbstractMap.java:418)
       org.elasticsearch.index.mapper.FieldTypeLookup.simpleMatchToFullName(FieldTypeLookup.java:219)
       org.elasticsearch.index.mapper.MapperService.simpleMatchToFullName(MapperService.java:783)
       org.elasticsearch.index.query.QueryShardContext.simpleMatchToIndexNames(QueryShardContext.java:215)
       org.elasticsearch.index.search.QueryParserHelper.resolveMappingField(QueryParserHelper.java:128)
       org.elasticsearch.index.search.QueryParserHelper.resolveMappingField(QueryParserHelper.java:110)
       org.elasticsearch.index.search.QueryStringQueryParser.<init>(QueryStringQueryParser.java:142)
       org.elasticsearch.index.query.QueryStringQueryBuilder.doToQuery(QueryStringQueryBuilder.java:1000)
       org.elasticsearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:105)
       org.elasticsearch.index.query.BoolQueryBuilder.addBooleanClauses(BoolQueryBuilder.java:415)
       org.elasticsearch.index.query.BoolQueryBuilder.doToQuery(BoolQueryBuilder.java:383)
       org.elasticsearch.index.query.AbstractQueryBuilder.toQuery(AbstractQueryBuilder.java:105)
       org.elasticsearch.index.query.QueryShardContext.lambda$toQuery$2(QueryShardContext.java:319)
       org.elasticsearch.index.query.QueryShardContext$$Lambda$3682/369120218.apply(Unknown Source)
       org.elasticsearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:331)
       org.elasticsearch.index.query.QueryShardContext.toQuery(QueryShardContext.java:318)
       org.elasticsearch.search.SearchService.parseSource(SearchService.java:791)
       org.elasticsearch.search.SearchService.createContext(SearchService.java:637)
       org.elasticsearch.search.SearchService.createAndPutContext(SearchService.java:596)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:387)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {client2}{LKveL5ccQHqXN2cP41U9wg}{wlObUGE3TAuk8stFALtjOQ}{192.168.2.237}{192.168.2.237:9301}{ml.machine_memory=25338519552, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
   Hot threads at 2019-07-20T13:42:30.463Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

::: {server0173-2}{56zRoXYuTsO9ssRPw21utw}{nR0ZFsgMTi-a--wjtwS0AQ}{192.168.3.126}{192.168.3.126:9302}{ml.machine_memory=405438406656, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.463Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   20.2% (101.2ms out of 500ms) cpu usage by thread 'elasticsearch[server0173-2][search][T#182]'
     3/10 snapshots sharing following 40 elements
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 40 elements
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.getFrame(SegmentTermsEnum.java:215)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.pushFrame(SegmentTermsEnum.java:241)
       org.apache.lucene.codecs.blocktree.SegmentTermsEnum.seekExact(SegmentTermsEnum.java:471)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:145)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 29 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

::: {server0300-5}{Kw49FSsUQNChffWPLhlWZQ}{l_QNxyqERy6O-IKJc6IE5A}{192.168.3.43}{192.168.3.43:9305}{ml.machine_memory=338702852096, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true, data=hot}
   Hot threads at 2019-07-20T13:42:30.462Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:

   101.6% (507.7ms out of 500ms) cpu usage by thread 'elasticsearch[server0300-5][search][T#21]'
     9/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     unique snapshot
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:150)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

   32.8% (164.1ms out of 500ms) cpu usage by thread 'elasticsearch[server0300-5][search][T#40]'
     3/10 snapshots sharing following 46 elements
       java.util.TreeMap.getEntry(TreeMap.java:359)
       java.util.TreeMap.get(TreeMap.java:278)
       org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:306)
       org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
       org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
       com.floragunn.searchguard.configuration.DlsFlsFilterLeafReader.terms(DlsFlsFilterLeafReader.java:907)
       org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:140)
       org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:108)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:725)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     4/10 snapshots sharing following 33 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)
     3/10 snapshots sharing following 36 elements
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorer(LRUQueryCache.java:780)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorer(IndicesQueryCache.java:151)
       org.apache.lucene.search.DisjunctionMaxQuery$DisjunctionMaxWeight.scorer(DisjunctionMaxQuery.java:136)
       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:143)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.scorerSupplier(LRUQueryCache.java:719)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.scorerSupplier(IndicesQueryCache.java:157)
       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:364)
       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:330)
       org.apache.lucene.search.Weight.bulkScorer(Weight.java:177)
       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:324)
       org.apache.lucene.search.LRUQueryCache$CachingWrapperWeight.bulkScorer(LRUQueryCache.java:800)
       org.elasticsearch.indices.IndicesQueryCache$CachingWeightWrapper.bulkScorer(IndicesQueryCache.java:163)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:667)
       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)
       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:471)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:276)
       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:114)
       org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:350)
       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:394)
       org.elasticsearch.search.SearchService.access$100(SearchService.java:126)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:359)
       org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:355)
       org.elasticsearch.search.SearchService$4.doRun(SearchService.java:1107)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
       java.lang.Thread.run(Thread.java:748)

...