API tools faq
paste
Advertisement
Guest User

hot_threads_20200905

a guest
Sep 5th, 2020
43
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 384.94 KB | None | 0 0
raw download clone embed print report
  1. ::: {chi1-lapp2682}{GQF8zBZ5TL-VxvxbZdY5NA}{tTQXNgCRSdGM6f1BtPY-mw}{10.128.170.78}{10.128.170.78:9300}{dil}{ml.machine_memory=126564622336, ml.max_open_jobs=20, xpack.installed=true}
  2. Hot threads at 2020-09-05T21:56:41.023Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3.  
  4. ::: {chi1-lapp3370}{7zfEswcjR9CjY7t8OvJGVg}{yaAQQY5zRgqmJ-wT7FB4Pg}{10.128.166.132}{10.128.166.132:9300}{dil}{ml.machine_memory=126564573184, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  5. Hot threads at 2020-09-05T21:56:41.035Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  6.  
  7. 78.2% (391.1ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3370][search][T#4]'
  8. 2/10 snapshots sharing following 38 elements
  9. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$3.longValue(Lucene80DocValuesProducer.java:475)
  10. app//org.elasticsearch.index.fielddata.SortableLongBitsToNumericDoubleValues.doubleValue(SortableLongBitsToNumericDoubleValues.java:42)
  11. app//org.elasticsearch.index.fielddata.SingletonSortedNumericDoubleValues.nextValue(SingletonSortedNumericDoubleValues.java:54)
  12. app//org.elasticsearch.index.fielddata.ScriptDocValues$Doubles.setNextDocId(ScriptDocValues.java:222)
  13. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:107)
  14. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:41)
  15. org.elasticsearch.painless.PainlessScript$Script.execute(double titlePopCount = doc['popularWordCountInTitle'].empty ? 0 : (double)doc['popularWordCountInTitle'].value; double desPopCount = doc['popularWordCountInDescription'].empty ? 0 : (double)doc['popularWordCountInDescription'].value; if (((3*titlePopCount) ...:28)
  16. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  17. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  18. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  19. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  20. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  21. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  22. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  23. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  24. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  25. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  26. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  27. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  28. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  29. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  30. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  31. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  32. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  33. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  34. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  35. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  36. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  37. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  38. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  39. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  40. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  41. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  42. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  43. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  44. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  45. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  46. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  47. 3/10 snapshots sharing following 31 elements
  48. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  49. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  50. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  51. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  52. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  53. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  54. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  55. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  56. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  57. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  58. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  59. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  60. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  61. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  62. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  63. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  64. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  65. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  66. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  67. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  68. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  69. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  70. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  71. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  72. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  73. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  74. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  75. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  76. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  77. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  78. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  79. 2/10 snapshots sharing following 25 elements
  80. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  81. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  82. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  83. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  84. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  85. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  86. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  87. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  88. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  89. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  90. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  91. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  92. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  93. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  94. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  95. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  96. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  97. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  98. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  99. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  100. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  101. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  102. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  103. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  104. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  105. 2/10 snapshots sharing following 24 elements
  106. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  107. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  108. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  109. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  110. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  111. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  112. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  113. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  114. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  115. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  116. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  117. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  118. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  119. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  120. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  121. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  122. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  123. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  124. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  125. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  126. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  127. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  128. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  129. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  130. unique snapshot
  131. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:57)
  132. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  133. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  134. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  135. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  136. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  137. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  138. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  139. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  140. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  141. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  142. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  143. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  144. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  145. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  146. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  147. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  148. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  149. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  150. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  151. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  152. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  153. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  154. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  155. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  156.  
  157. 62.0% (309.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3370][search][T#23]'
  158. 9/10 snapshots sharing following 21 elements
  159. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  160. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  161. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  162. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  163. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  164. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  165. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  166. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  167. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  168. app//org.elasticsearch.search.SearchService$$Lambda$4564/0x00000008018a7840.get(Unknown Source)
  169. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  170. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  171. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  172. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  173. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  174. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  175. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  176. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  177. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  178. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  179. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  180.  
  181. 60.4% (301.8ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3370][search][T#18]'
  182. 6/10 snapshots sharing following 40 elements
  183. app//com.fasterxml.jackson.core.util.InternCache.intern(InternCache.java:55)
  184. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.addName(ByteQuadsCanonicalizer.java:808)
  185. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.addName(UTF8StreamJsonParser.java:2394)
  186. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.findName(UTF8StreamJsonParser.java:2278)
  187. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1885)
  188. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1840)
  189. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1797)
  190. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1732)
  191. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:776)
  192. app//org.elasticsearch.common.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:52)
  193. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:337)
  194. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  195. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  196. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  197. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162a840.apply(Unknown Source)
  198. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  199. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  200. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  201. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  202. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  203. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  204. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  205. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  206. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  207. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  208. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  209. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  210. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  211. app//org.elasticsearch.search.SearchService$$Lambda$4616/0x00000008018f0040.get(Unknown Source)
  212. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  213. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  214. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  215. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  216. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  217. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  218. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  219. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  220. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  221. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  222. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  223. 2/10 snapshots sharing following 43 elements
  224. app//com.fasterxml.jackson.core.util.InternCache.intern(InternCache.java:55)
  225. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.addName(ByteQuadsCanonicalizer.java:808)
  226. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.rehash(ByteQuadsCanonicalizer.java:1129)
  227. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer._verifySharing(ByteQuadsCanonicalizer.java:877)
  228. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.addName(ByteQuadsCanonicalizer.java:806)
  229. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.addName(UTF8StreamJsonParser.java:2394)
  230. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.findName(UTF8StreamJsonParser.java:2278)
  231. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1885)
  232. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1840)
  233. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1797)
  234. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1732)
  235. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:776)
  236. app//org.elasticsearch.common.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:52)
  237. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:337)
  238. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  239. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  240. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  241. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162a840.apply(Unknown Source)
  242. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  243. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  244. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  245. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  246. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  247. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  248. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  249. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  250. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  251. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  252. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  253. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  254. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  255. app//org.elasticsearch.search.SearchService$$Lambda$4616/0x00000008018f0040.get(Unknown Source)
  256. app//org.elasticsearch.search.SearchService$$Lambda$4565/0x00000008018a7c40.get(Unknown Source)
  257. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  258. app//org.elasticsearch.action.ActionRunnable$$Lambda$4483/0x0000000801889840.accept(Unknown Source)
  259. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  260. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  261. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  262. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  263. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  264. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  265. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  266. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  267.  
  268. ::: {chi1-lapp3134}{9nLZKH3WQWm9pSBLKWI4qQ}{rdxQzh0rSpWDWxuPjIKgjA}{10.128.177.135}{10.128.177.135:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  269. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  270.  
  271. 45.1% (225.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3134][search][T#15]'
  272. 7/10 snapshots sharing following 2 elements
  273. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  274. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  275.  
  276. ::: {chi1-lapp3128}{eidz7CUNSWmhsicY_zKNDg}{3Mw0w1AES-KsLDiLA9kEHw}{10.128.177.197}{10.128.177.197:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  277. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  278.  
  279. ::: {chi1-lapp1521}{qPxhTW6MQ-6_wa8hp5vjxg}{kgmx47_6RFqkfVhV7l2emw}{10.128.176.71}{10.128.176.71:9300}{dil}{ml.machine_memory=126564769792, ml.max_open_jobs=20, xpack.installed=true}
  280. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  281.  
  282. 20.8% (104ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1521][[allgames_v8][9]: Lucene Merge Thread #358]'
  283. 3/10 snapshots sharing following 16 elements
  284. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.seekExact(Lucene80DocValuesProducer.java:987)
  285. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:909)
  286. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  287. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  288. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:491)
  289. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  290. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  291. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  292. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  293. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  294. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  295. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  296. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  297. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  298. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  299. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  300. 4/10 snapshots sharing following 20 elements
  301. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  302. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  303. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  304. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  305. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  306. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  307. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  308. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  309. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:508)
  310. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  311. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  312. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  313. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  314. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  315. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  316. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  317. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  318. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  319. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  320. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  321. 3/10 snapshots sharing following 11 elements
  322. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  323. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  324. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  325. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  326. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  327. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  328. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  329. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  330. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  331. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  332. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  333.  
  334. ::: {chi1-lapp3397}{nyRdKKOoRP2oJFeqKYNwzg}{1cp6AHAbRC225H1TXjQGIQ}{10.128.168.69}{10.128.168.69:9300}{dil}{ml.machine_memory=126564573184, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  335. Hot threads at 2020-09-05T21:56:41.078Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  336.  
  337. 88.3% (441.5ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3397][search][T#43]'
  338. 2/10 snapshots sharing following 24 elements
  339. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  340. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  341. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  342. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  343. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  344. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  345. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  346. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  347. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  348. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  349. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  350. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  351. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  352. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  353. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  354. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  355. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  356. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  357. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  358. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  359. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  360. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  361. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  362. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  363. 2/10 snapshots sharing following 36 elements
  364. app//org.apache.lucene.index.SingletonSortedNumericDocValues.advanceExact(SingletonSortedNumericDocValues.java:63)
  365. app//org.elasticsearch.index.fielddata.ScriptDocValues$Longs.setNextDocId(ScriptDocValues.java:95)
  366. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:107)
  367. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:41)
  368. org.elasticsearch.painless.PainlessScript$Script.execute(double titlePopCount = doc['popularWordCountInTitle'].empty ? 0 : (double)doc['popularWordCountInTitle'].value; double desPopCount = doc['popularWordCountInDescription'].empty ? 0 : (double)doc['popularWordCountInDescription'].value; if (((3*titlePopCount) ...:322)
  369. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  370. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  371. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  372. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  373. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  374. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  375. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  376. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  377. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  378. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  379. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  380. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  381. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  382. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  383. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  384. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  385. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  386. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  387. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  388. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  389. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  390. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  391. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  392. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  393. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  394. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  395. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  396. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  397. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  398. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  399. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  400. 6/10 snapshots sharing following 31 elements
  401. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  402. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  403. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  404. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  405. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  406. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  407. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  408. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  409. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  410. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  411. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  412. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  413. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  414. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  415. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  416. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  417. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  418. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  419. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  420. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  421. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  422. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  423. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  424. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  425. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  426. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  427. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  428. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  429. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  430. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  431. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  432.  
  433. 87.2% (436.1ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3397][search][T#16]'
  434. 2/10 snapshots sharing following 31 elements
  435. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  436. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  437. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  438. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  439. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  440. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  441. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  442. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  443. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  444. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  445. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  446. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  447. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  448. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  449. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  450. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  451. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  452. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  453. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  454. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  455. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  456. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  457. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  458. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  459. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  460. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  461. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  462. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  463. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  464. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  465. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  466. 4/10 snapshots sharing following 11 elements
  467. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  468. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  469. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  470. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  471. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  472. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  473. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  474. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  475. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  476. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  477. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  478. 4/10 snapshots sharing following 10 elements
  479. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  480. java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  481. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
  482. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  483. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
  484. app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
  485. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
  486. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  487. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  488. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  489.  
  490. 76.0% (379.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3397][search][T#23]'
  491. 6/10 snapshots sharing following 25 elements
  492. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  493. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  494. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  495. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  496. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  497. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  498. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  499. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  500. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  501. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  502. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  503. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  504. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  505. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  506. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  507. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  508. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  509. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  510. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  511. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  512. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  513. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  514. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  515. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  516. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  517. 3/10 snapshots sharing following 26 elements
  518. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  519. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  520. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  521. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  522. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  523. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  524. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  525. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  526. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  527. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  528. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  529. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  530. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  531. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  532. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  533. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  534. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  535. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  536. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  537. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  538. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  539. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  540. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  541. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  542. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  543. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  544. unique snapshot
  545. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  546. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  547. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  548. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  549. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  550. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  551. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  552. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  553. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  554. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  555. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  556. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  557. app//org.elasticsearch.search.SearchService$$Lambda$4554/0x00000008018a7040.get(Unknown Source)
  558. app//org.elasticsearch.search.SearchService$$Lambda$4555/0x00000008018a7440.get(Unknown Source)
  559. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  560. app//org.elasticsearch.action.ActionRunnable$$Lambda$4487/0x0000000801894c40.accept(Unknown Source)
  561. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  562. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  563. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  564. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  565. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  566. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  567. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  568. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  569.  
  570. ::: {chi1-lapp2893}{8Ve6uXc1R9-5aCBN0nPw4Q}{sinW7DrPQ9-NhWYwSABoxQ}{10.128.177.195}{10.128.177.195:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  571. Hot threads at 2020-09-05T21:56:41.025Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  572.  
  573. 33.8% (168.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2893][refresh][T#3]'
  574. 4/10 snapshots sharing following 34 elements
  575. app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:128)
  576. app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:121)
  577. app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:185)
  578. app//org.apache.lucene.index.TermsHashPerField.sortPostings(TermsHashPerField.java:97)
  579. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:79)
  580. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  581. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  582. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  583. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  584. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  585. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  586. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  587. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  588. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  589. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  590. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  591. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  592. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  593. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  594. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  595. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  596. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  597. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  598. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  599. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  600. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  601. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  602. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  603. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  604. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  605. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  606. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  607. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  608. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  609. 6/10 snapshots sharing following 31 elements
  610. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  611. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  612. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  613. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  614. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  615. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  616. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  617. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  618. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  619. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  620. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  621. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  622. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  623. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  624. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  625. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  626. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  627. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  628. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  629. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  630. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  631. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  632. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  633. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  634. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  635. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  636. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  637. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  638. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  639. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  640. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  641.  
  642. ::: {chi1-lapp2913}{1dAEqxRrRm2954bQ3dKZew}{zwh1Gdo0SXCb4lWyRzvkww}{10.128.177.133}{10.128.177.133:9300}{il}{ml.machine_memory=126564614144, xpack.installed=true, ml.max_open_jobs=20}
  643. Hot threads at 2020-09-05T21:56:41.023Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  644.  
  645. ::: {chi1-lapp3125}{n2hnRV-rQ6SNoTEX8cX3rw}{qLb2js-LQgKTik5tc5xDHA}{10.128.179.134}{10.128.179.134:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  646. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  647.  
  648. ::: {chi1-lapp2933}{ZBSwxJZVR3Cz2a1YX0BC7g}{FE_iIk3aRHCyLBByqYtyfg}{10.128.131.205}{10.128.131.205:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  649. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  650.  
  651. 100.4% (501.8ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2933][[allgames_v5][6]: Lucene Merge Thread #147643]'
  652. 4/10 snapshots sharing following 18 elements
  653. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.next(Lucene80DocValuesProducer.java:972)
  654. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.seekExact(Lucene80DocValuesProducer.java:987)
  655. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:909)
  656. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  657. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  658. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeTermsIndex(Lucene80DocValuesConsumer.java:541)
  659. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:524)
  660. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  661. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  662. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  663. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  664. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  665. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  666. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  667. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  668. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  669. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  670. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  671. 2/10 snapshots sharing following 18 elements
  672. app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
  673. app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
  674. app//org.apache.lucene.index.OrdinalMap.getFirstSegmentOrd(OrdinalMap.java:336)
  675. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:776)
  676. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  677. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeTermsIndex(Lucene80DocValuesConsumer.java:541)
  678. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:524)
  679. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  680. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  681. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  682. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  683. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  684. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  685. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  686. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  687. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  688. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  689. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  690. 4/10 snapshots sharing following 14 elements
  691. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  692. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.writeTermsIndex(Lucene80DocValuesConsumer.java:541)
  693. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:524)
  694. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  695. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  696. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  697. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  698. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  699. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  700. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  701. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  702. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  703. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  704. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  705.  
  706. 86.9% (434.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2933][[allgames_v8][0]: Lucene Merge Thread #342]'
  707. 4/10 snapshots sharing following 13 elements
  708. app//org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:127)
  709. app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:129)
  710. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:865)
  711. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  712. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  713. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  714. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  715. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  716. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  717. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  718. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  719. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  720. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  721. 2/10 snapshots sharing following 24 elements
  722. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  723. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  724. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  725. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  726. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  727. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  728. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  729. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  730. app//org.apache.lucene.store.DataOutput.writeBytes(DataOutput.java:52)
  731. app//org.apache.lucene.store.RAMOutputStream.writeTo(RAMOutputStream.java:90)
  732. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlock(BlockTreeTermsWriter.java:826)
  733. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:603)
  734. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  735. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  736. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  737. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  738. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  739. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  740. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  741. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  742. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  743. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  744. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  745. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  746. 4/10 snapshots sharing following 9 elements
  747. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  748. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  749. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  750. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  751. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  752. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  753. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  754. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  755. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  756.  
  757. ::: {chi1-lapp3119}{UL5TnzBjRnC-fH8hBqT_9Q}{eHOaVnbCT3-cLhZmbdY7KA}{10.128.178.71}{10.128.178.71:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  758. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  759.  
  760. 57.1% (285.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3119][search][T#28]'
  761. 3/10 snapshots sharing following 17 elements
  762. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  763. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  764. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  765. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  766. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  767. app//org.elasticsearch.search.SearchService$$Lambda$4840/0x0000000801977c40.get(Unknown Source)
  768. app//org.elasticsearch.search.SearchService$$Lambda$4841/0x0000000801977040.get(Unknown Source)
  769. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  770. app//org.elasticsearch.action.ActionRunnable$$Lambda$4658/0x0000000801945840.accept(Unknown Source)
  771. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  772. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  773. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  774. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  775. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  776. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  777. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  778. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  779.  
  780. 29.1% (145.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3119][search][T#23]'
  781. 2/10 snapshots sharing following 25 elements
  782. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  783. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  784. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  785. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  786. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  787. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  788. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  789. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  790. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  791. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  792. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  793. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  794. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  795. app//org.elasticsearch.search.SearchService$$Lambda$4840/0x0000000801977c40.get(Unknown Source)
  796. app//org.elasticsearch.search.SearchService$$Lambda$4841/0x0000000801977040.get(Unknown Source)
  797. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  798. app//org.elasticsearch.action.ActionRunnable$$Lambda$4658/0x0000000801945840.accept(Unknown Source)
  799. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  800. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  801. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  802. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  803. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  804. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  805. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  806. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  807.  
  808. ::: {chi1-lapp2668}{qCIs4WVNRC-Pn-wx3gHZuQ}{3DupyKcFRMiLQ0QaGfYGIA}{10.128.153.144}{10.128.153.144:9300}{dil}{ml.machine_memory=126564622336, ml.max_open_jobs=20, xpack.installed=true}
  809. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  810.  
  811. 55.5% (277.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2668][refresh][T#1]'
  812. 3/10 snapshots sharing following 31 elements
  813. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  814. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  815. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  816. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  817. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  818. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  819. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  820. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  821. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  822. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  823. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  824. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  825. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  826. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  827. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  828. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  829. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  830. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  831. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  832. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  833. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  834. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  835. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  836. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  837. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  838. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  839. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  840. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  841. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  842. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  843. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  844. 2/10 snapshots sharing following 31 elements
  845. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  846. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  847. app//org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
  848. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
  849. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  850. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  851. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  852. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  853. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  854. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  855. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  856. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  857. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  858. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  859. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  860. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  861. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  862. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  863. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  864. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  865. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  866. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  867. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  868. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  869. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  870. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  871. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  872. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  873. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  874. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  875. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  876. 2/10 snapshots sharing following 24 elements
  877. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  878. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  879. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  880. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  881. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  882. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  883. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  884. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  885. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  886. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  887. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  888. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  889. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  890. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  891. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  892. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  893. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  894. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  895. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  896. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  897. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  898. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  899. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  900. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  901. 3/10 snapshots sharing following 9 elements
  902. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  903. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  904. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
  905. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  906. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
  907. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
  908. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  909. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  910. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  911.  
  912. ::: {chi1-lapp3133}{Lc6oPdRmQcyzx_NEw3lggA}{WStgY-uzT3GMNvemRw29oA}{10.128.135.6}{10.128.135.6:9300}{lm}{ml.machine_memory=8191987712, ml.max_open_jobs=20, xpack.installed=true}
  913. Hot threads at 2020-09-05T21:56:41.023Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  914.  
  915. 0.8% (4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3133][transport_worker][T#1]'
  916. 5/10 snapshots sharing following 3 elements
  917. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  918. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  919. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  920.  
  921. ::: {chi1-lapp2676}{XZhXliOFTbiNtTF1PXT2zQ}{RymAEBMWTJCpDgm7Z3KsyQ}{10.128.169.204}{10.128.169.204:9300}{dil}{ml.machine_memory=126564626432, ml.max_open_jobs=20, xpack.installed=true}
  922. Hot threads at 2020-09-05T21:56:41.024Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  923.  
  924. 8.7% (43.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2676][refresh][T#3]'
  925. 2/10 snapshots sharing following 37 elements
  926. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.append(BlockTreeTermsWriter.java:502)
  927. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:474)
  928. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:634)
  929. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  930. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  931. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  932. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  933. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  934. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  935. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  936. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  937. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  938. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  939. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  940. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  941. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  942. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  943. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  944. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  945. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  946. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  947. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  948. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  949. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  950. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  951. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  952. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  953. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  954. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  955. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  956. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  957. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  958. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  959. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  960. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  961. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  962. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  963. 2/10 snapshots sharing following 24 elements
  964. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  965. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  966. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  967. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  968. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  969. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  970. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  971. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  972. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  973. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  974. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  975. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  976. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  977. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  978. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  979. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  980. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  981. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  982. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  983. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  984. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  985. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  986. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  987. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  988. 6/10 snapshots sharing following 9 elements
  989. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  990. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  991. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
  992. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  993. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
  994. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
  995. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  996. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  997. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  998.  
  999. 1.1% (5.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2676][[allgames_v5][2]: Lucene Merge Thread #148961]'
  1000. 10/10 snapshots sharing following 16 elements
  1001. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  1002. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  1003. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  1004. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  1005. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  1006. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  1007. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  1008. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  1009. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  1010. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  1011. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  1012. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527)
  1013. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1014. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1015. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1016. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1017.  
  1018. ::: {chi1-lapp3367}{eH9UMPF3TROMlNZxfAcExA}{1qvzidMKQjiNNpC8EqAdaw}{10.128.167.68}{10.128.167.68:9300}{dil}{ml.machine_memory=126564569088, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  1019. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1020.  
  1021. 23.3% (116.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3367][[allgames_v5][2]: Lucene Merge Thread #120]'
  1022. 10/10 snapshots sharing following 12 elements
  1023. app//org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  1024. app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  1025. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:238)
  1026. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  1027. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:645)
  1028. app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
  1029. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
  1030. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  1031. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1032. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1033. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1034. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1035.  
  1036. 1.2% (5.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3367][[allgames_v8][4]: Lucene Merge Thread #337]'
  1037. 10/10 snapshots sharing following 16 elements
  1038. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  1039. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  1040. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  1041. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  1042. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  1043. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  1044. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  1045. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  1046. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  1047. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  1048. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  1049. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527)
  1050. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1051. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1052. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1053. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1054.  
  1055. ::: {chi1-lapp3135}{vuB74OtNQBKnSLz0hC7RSA}{xx-qpV4uTFq_oIdqER75XQ}{10.128.178.73}{10.128.178.73:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  1056. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1057.  
  1058. 24.2% (121ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3135][[allgames_v5][7]: Lucene Merge Thread #120]'
  1059. 10/10 snapshots sharing following 10 elements
  1060. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  1061. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  1062. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  1063. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  1064. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  1065. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  1066. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1067. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1068. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1069. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1070.  
  1071. 22.4% (112.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3135][write][T#19]'
  1072. 2/10 snapshots sharing following 42 elements
  1073. app//org.apache.lucene.index.SortedSetDocValuesWriter.addOneValue(SortedSetDocValuesWriter.java:124)
  1074. app//org.apache.lucene.index.SortedSetDocValuesWriter.addValue(SortedSetDocValuesWriter.java:90)
  1075. app//org.apache.lucene.index.DefaultIndexingChain.indexDocValue(DefaultIndexingChain.java:628)
  1076. app//org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:479)
  1077. app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:406)
  1078. app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
  1079. app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
  1080. app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
  1081. app//org.apache.lucene.index.IndexWriter.softUpdateDocument(IndexWriter.java:1654)
  1082. app//org.elasticsearch.index.engine.InternalEngine.updateDocs(InternalEngine.java:1259)
  1083. app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1090)
  1084. app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:934)
  1085. app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:813)
  1086. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:785)
  1087. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:749)
  1088. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:455)
  1089. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:437)
  1090. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:408)
  1091. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:81)
  1092. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:513)
  1093. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:487)
  1094. app//org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$30(IndexShard.java:2984)
  1095. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4372/0x00000008018ca440.accept(Unknown Source)
  1096. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  1097. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  1098. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  1099. app//org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$28(IndexShard.java:2923)
  1100. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4371/0x00000008018cac40.accept(Unknown Source)
  1101. app//org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:3023)
  1102. app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2922)
  1103. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:849)
  1104. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:585)
  1105. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1106. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:472)
  1107. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2877/0x0000000801512440.messageReceived(Unknown Source)
  1108. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  1109. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  1110. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1111. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1112. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1113. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1114. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1115. 2/10 snapshots sharing following 37 elements
  1116. app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
  1117. app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
  1118. app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
  1119. app//org.apache.lucene.index.IndexWriter.softUpdateDocument(IndexWriter.java:1654)
  1120. app//org.elasticsearch.index.engine.InternalEngine.updateDocs(InternalEngine.java:1259)
  1121. app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1090)
  1122. app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:934)
  1123. app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:813)
  1124. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:785)
  1125. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:749)
  1126. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:455)
  1127. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:437)
  1128. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:408)
  1129. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:81)
  1130. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:513)
  1131. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:487)
  1132. app//org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$30(IndexShard.java:2984)
  1133. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4372/0x00000008018ca440.accept(Unknown Source)
  1134. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  1135. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  1136. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  1137. app//org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$28(IndexShard.java:2923)
  1138. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4371/0x00000008018cac40.accept(Unknown Source)
  1139. app//org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:3023)
  1140. app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2922)
  1141. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:849)
  1142. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:585)
  1143. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1144. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:472)
  1145. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2877/0x0000000801512440.messageReceived(Unknown Source)
  1146. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  1147. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  1148. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1149. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1150. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1151. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1152. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1153. 6/10 snapshots sharing following 2 elements
  1154. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1155. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1156.  
  1157. ::: {chi1-lapp3118}{NNTR1dgnRCaFO_Gmc_6e7w}{qwOU6eG2T-Ci8d8VzbDsvQ}{10.128.178.72}{10.128.178.72:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  1158. Hot threads at 2020-09-05T21:56:41.025Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1159.  
  1160. 1.5% (7.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3118][[allgames_v8][2]: Lucene Merge Thread #318]'
  1161. 10/10 snapshots sharing following 16 elements
  1162. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  1163. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  1164. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  1165. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  1166. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  1167. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  1168. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  1169. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  1170. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  1171. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  1172. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  1173. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527)
  1174. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1175. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1176. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1177. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1178.  
  1179. ::: {chi1-lapp3366}{AvBNBGUJSs-FpiIeJ-loZA}{J6gGR13-RJe1zozVihLZrA}{10.128.168.132}{10.128.168.132:9300}{dil}{ml.machine_memory=126564564992, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  1180. Hot threads at 2020-09-05T21:56:41.063Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1181.  
  1182. 79.7% (398.5ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3366][search][T#20]'
  1183. 3/10 snapshots sharing following 31 elements
  1184. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  1185. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  1186. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  1187. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  1188. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  1189. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  1190. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  1191. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  1192. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  1193. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1194. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1195. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1196. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1197. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1198. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1199. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1200. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1201. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1202. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1203. app//org.elasticsearch.search.SearchService$$Lambda$4515/0x000000080189f040.get(Unknown Source)
  1204. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1205. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1206. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1207. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1208. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1209. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1210. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1211. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1212. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1213. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1214. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1215. 7/10 snapshots sharing following 22 elements
  1216. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1217. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1218. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1219. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1220. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1221. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1222. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1223. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1224. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1225. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1226. app//org.elasticsearch.search.SearchService$$Lambda$4515/0x000000080189f040.get(Unknown Source)
  1227. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1228. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1229. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1230. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1231. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1232. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1233. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1234. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1235. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1236. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1237. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1238.  
  1239. 79.1% (395.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3366][search][T#34]'
  1240. 6/10 snapshots sharing following 40 elements
  1241. app//com.fasterxml.jackson.core.util.InternCache.intern(InternCache.java:55)
  1242. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.addName(ByteQuadsCanonicalizer.java:808)
  1243. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.addName(UTF8StreamJsonParser.java:2394)
  1244. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.findName(UTF8StreamJsonParser.java:2278)
  1245. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1885)
  1246. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1840)
  1247. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1797)
  1248. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1732)
  1249. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:776)
  1250. app//org.elasticsearch.common.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:52)
  1251. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:337)
  1252. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1253. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1254. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1255. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162bc40.apply(Unknown Source)
  1256. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1257. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1258. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  1259. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  1260. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  1261. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  1262. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  1263. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  1264. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  1265. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  1266. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  1267. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  1268. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1269. app//org.elasticsearch.search.SearchService$$Lambda$4608/0x00000008018fe440.get(Unknown Source)
  1270. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1271. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1272. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1273. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1274. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1275. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1276. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1277. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1278. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1279. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1280. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1281. 2/10 snapshots sharing following 46 elements
  1282. java.base@13.0.2/java.util.HashMap.put(HashMap.java:612)
  1283. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:343)
  1284. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1285. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1286. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1287. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162bc40.apply(Unknown Source)
  1288. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1289. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1290. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1291. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1292. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162bc40.apply(Unknown Source)
  1293. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1294. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1295. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1296. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1297. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162bc40.apply(Unknown Source)
  1298. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1299. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1300. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1301. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1302. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162bc40.apply(Unknown Source)
  1303. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1304. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1305. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  1306. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  1307. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  1308. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  1309. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  1310. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  1311. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  1312. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  1313. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  1314. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  1315. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1316. app//org.elasticsearch.search.SearchService$$Lambda$4608/0x00000008018fe440.get(Unknown Source)
  1317. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1318. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1319. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1320. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1321. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1322. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1323. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1324. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1325. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1326. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1327. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1328. 2/10 snapshots sharing following 11 elements
  1329. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1330. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1331. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1332. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1333. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1334. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1335. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1336. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1337. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1338. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1339. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1340.  
  1341. 78.7% (393.7ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3366][search][T#15]'
  1342. 10/10 snapshots sharing following 13 elements
  1343. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1344. app//org.elasticsearch.search.SearchService$$Lambda$4608/0x00000008018fe440.get(Unknown Source)
  1345. app//org.elasticsearch.search.SearchService$$Lambda$4516/0x000000080189f440.get(Unknown Source)
  1346. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1347. app//org.elasticsearch.action.ActionRunnable$$Lambda$4450/0x0000000801882c40.accept(Unknown Source)
  1348. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1349. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1350. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1351. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1352. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1353. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1354. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1355. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1356.  
  1357. ::: {chi1-lapp3137}{MhdQaZVIRr-Ojr9e1diD_A}{kVtYS8b1TdqZtClaIW2oJQ}{10.128.177.198}{10.128.177.198:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  1358. Hot threads at 2020-09-05T21:56:41.025Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1359.  
  1360. 22.6% (112.7ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3137][write][T#21]'
  1361. 2/10 snapshots sharing following 38 elements
  1362. app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:406)
  1363. app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
  1364. app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
  1365. app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
  1366. app//org.apache.lucene.index.IndexWriter.softUpdateDocument(IndexWriter.java:1654)
  1367. app//org.elasticsearch.index.engine.InternalEngine.updateDocs(InternalEngine.java:1259)
  1368. app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1090)
  1369. app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:934)
  1370. app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:813)
  1371. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:785)
  1372. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:749)
  1373. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:455)
  1374. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:437)
  1375. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:408)
  1376. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:81)
  1377. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:513)
  1378. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:487)
  1379. app//org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$30(IndexShard.java:2984)
  1380. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4323/0x0000000801889c40.accept(Unknown Source)
  1381. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  1382. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  1383. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  1384. app//org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$28(IndexShard.java:2923)
  1385. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4322/0x000000080188a440.accept(Unknown Source)
  1386. app//org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:3023)
  1387. app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2922)
  1388. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:849)
  1389. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:585)
  1390. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1391. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:472)
  1392. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2877/0x0000000801509c40.messageReceived(Unknown Source)
  1393. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  1394. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  1395. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1396. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1397. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1398. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1399. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1400. 2/10 snapshots sharing following 44 elements
  1401. app//org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  1402. app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  1403. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:238)
  1404. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  1405. app//org.apache.lucene.index.StoredFieldsConsumer.finishDocument(StoredFieldsConsumer.java:59)
  1406. app//org.apache.lucene.index.DefaultIndexingChain.finishStoredFields(DefaultIndexingChain.java:378)
  1407. app//org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:414)
  1408. app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
  1409. app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
  1410. app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
  1411. app//org.apache.lucene.index.IndexWriter.softUpdateDocument(IndexWriter.java:1654)
  1412. app//org.elasticsearch.index.engine.InternalEngine.updateDocs(InternalEngine.java:1259)
  1413. app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1090)
  1414. app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:934)
  1415. app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:813)
  1416. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:785)
  1417. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:749)
  1418. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:455)
  1419. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:437)
  1420. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:408)
  1421. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:81)
  1422. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:513)
  1423. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:487)
  1424. app//org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$30(IndexShard.java:2984)
  1425. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4323/0x0000000801889c40.accept(Unknown Source)
  1426. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  1427. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  1428. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  1429. app//org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$28(IndexShard.java:2923)
  1430. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4322/0x000000080188a440.accept(Unknown Source)
  1431. app//org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:3023)
  1432. app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2922)
  1433. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:849)
  1434. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:585)
  1435. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1436. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:472)
  1437. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2877/0x0000000801509c40.messageReceived(Unknown Source)
  1438. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  1439. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  1440. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1441. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1442. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1443. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1444. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1445. 6/10 snapshots sharing following 2 elements
  1446. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1447. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1448.  
  1449. 10.2% (50.8ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3137][write][T#10]'
  1450. 10/10 snapshots sharing following 28 elements
  1451. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnReplica(IndexShard.java:749)
  1452. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOpOnReplica(TransportShardBulkAction.java:455)
  1453. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnReplica(TransportShardBulkAction.java:437)
  1454. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:408)
  1455. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:81)
  1456. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:513)
  1457. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:487)
  1458. app//org.elasticsearch.index.shard.IndexShard.lambda$innerAcquireReplicaOperationPermit$30(IndexShard.java:2984)
  1459. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4323/0x0000000801889c40.accept(Unknown Source)
  1460. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  1461. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  1462. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  1463. app//org.elasticsearch.index.shard.IndexShard.lambda$acquireReplicaOperationPermit$28(IndexShard.java:2923)
  1464. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4322/0x000000080188a440.accept(Unknown Source)
  1465. app//org.elasticsearch.index.shard.IndexShard.innerAcquireReplicaOperationPermit(IndexShard.java:3023)
  1466. app//org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationPermit(IndexShard.java:2922)
  1467. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquireReplicaOperationPermit(TransportReplicationAction.java:849)
  1468. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:585)
  1469. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1470. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handleReplicaRequest(TransportReplicationAction.java:472)
  1471. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2877/0x0000000801509c40.messageReceived(Unknown Source)
  1472. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  1473. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  1474. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1475. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1476. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1477. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1478. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1479.  
  1480. ::: {chi1-lapp3127}{25aOQvEMRsyRw-3tb87mQA}{xdzs_3trQD2liny4aqcWeg}{10.128.176.74}{10.128.176.74:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  1481. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1482.  
  1483. 14.3% (71.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3127][[allgames_v8][4]: Lucene Merge Thread #305]'
  1484. 2/10 snapshots sharing following 17 elements
  1485. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.next(Lucene80DocValuesProducer.java:972)
  1486. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.seekExact(Lucene80DocValuesProducer.java:987)
  1487. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:909)
  1488. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  1489. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  1490. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:491)
  1491. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  1492. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  1493. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  1494. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  1495. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  1496. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  1497. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  1498. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1499. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1500. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1501. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1502. 2/10 snapshots sharing following 20 elements
  1503. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  1504. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  1505. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  1506. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  1507. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  1508. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  1509. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  1510. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  1511. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:495)
  1512. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  1513. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  1514. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  1515. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  1516. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  1517. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  1518. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  1519. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1520. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1521. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1522. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1523. 6/10 snapshots sharing following 20 elements
  1524. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  1525. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  1526. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  1527. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  1528. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  1529. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  1530. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  1531. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  1532. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:508)
  1533. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  1534. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  1535. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  1536. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  1537. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  1538. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  1539. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  1540. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  1541. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  1542. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  1543. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  1544.  
  1545. ::: {chi1-lapp3120}{mw5ni4RZTTq1d8FZwHsmyA}{ia-AQ6sTT4-DrCAhMYXwQw}{10.128.176.132}{10.128.176.132:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  1546. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1547.  
  1548. 79.7% (398.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3120][search][T#17]'
  1549. 7/10 snapshots sharing following 13 elements
  1550. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1551. app//org.elasticsearch.search.SearchService$$Lambda$4487/0x0000000801947840.get(Unknown Source)
  1552. app//org.elasticsearch.search.SearchService$$Lambda$4465/0x0000000801915c40.get(Unknown Source)
  1553. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1554. app//org.elasticsearch.action.ActionRunnable$$Lambda$4451/0x0000000801913040.accept(Unknown Source)
  1555. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1556. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1557. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1558. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1559. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1560. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1561. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1562. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1563.  
  1564. 69.8% (349.1ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3120][refresh][T#3]'
  1565. 2/10 snapshots sharing following 33 elements
  1566. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:865)
  1567. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  1568. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  1569. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  1570. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  1571. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  1572. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  1573. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  1574. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  1575. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  1576. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  1577. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  1578. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  1579. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  1580. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  1581. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  1582. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1583. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  1584. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  1585. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  1586. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1587. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  1588. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  1589. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  1590. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  1591. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  1592. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  1593. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  1594. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  1595. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  1596. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1597. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1598. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1599. 3/10 snapshots sharing following 35 elements
  1600. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:625)
  1601. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  1602. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  1603. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  1604. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  1605. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  1606. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  1607. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  1608. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  1609. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  1610. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  1611. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  1612. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  1613. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  1614. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  1615. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  1616. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  1617. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  1618. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1619. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  1620. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  1621. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  1622. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1623. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  1624. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  1625. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  1626. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  1627. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  1628. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  1629. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  1630. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  1631. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  1632. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1633. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1634. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1635. 2/10 snapshots sharing following 33 elements
  1636. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  1637. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  1638. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  1639. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  1640. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  1641. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  1642. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  1643. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  1644. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  1645. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  1646. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  1647. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  1648. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  1649. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  1650. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  1651. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  1652. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1653. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  1654. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  1655. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  1656. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1657. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  1658. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  1659. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  1660. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  1661. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  1662. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  1663. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  1664. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  1665. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  1666. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1667. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1668. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1669. 2/10 snapshots sharing following 33 elements
  1670. app//org.apache.lucene.store.BufferedChecksumIndexInput.readBytes(BufferedChecksumIndexInput.java:49)
  1671. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:277)
  1672. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  1673. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  1674. app//org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
  1675. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
  1676. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  1677. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  1678. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  1679. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  1680. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  1681. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  1682. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  1683. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  1684. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  1685. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  1686. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1687. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  1688. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  1689. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  1690. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1691. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  1692. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  1693. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  1694. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  1695. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  1696. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  1697. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  1698. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  1699. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  1700. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1701. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1702. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1703. unique snapshot
  1704. java.base@13.0.2/sun.nio.ch.FileDispatcherImpl.write0(Native Method)
  1705. java.base@13.0.2/sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:62)
  1706. java.base@13.0.2/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:113)
  1707. java.base@13.0.2/sun.nio.ch.IOUtil.write(IOUtil.java:79)
  1708. java.base@13.0.2/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:280)
  1709. java.base@13.0.2/java.nio.channels.Channels.writeFullyImpl(Channels.java:74)
  1710. java.base@13.0.2/java.nio.channels.Channels.writeFully(Channels.java:97)
  1711. java.base@13.0.2/java.nio.channels.Channels$1.write(Channels.java:172)
  1712. app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416)
  1713. java.base@13.0.2/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74)
  1714. java.base@13.0.2/java.io.BufferedOutputStream.write(BufferedOutputStream.java:123)
  1715. app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
  1716. app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59)
  1717. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  1718. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  1719. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  1720. app//org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:574)
  1721. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:513)
  1722. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  1723. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  1724. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  1725. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  1726. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  1727. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  1728. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  1729. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  1730. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  1731. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  1732. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1733. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  1734. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  1735. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  1736. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  1737. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  1738. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  1739. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  1740. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  1741. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  1742. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  1743. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  1744. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  1745. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  1746. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1747. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1748. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1749.  
  1750. ::: {chi1-lapp3373}{VMi4iEhpS4Kg6hQ081OTEA}{OmsmvIC3S0qKJquxWab2iw}{10.128.169.3}{10.128.169.3:9300}{dil}{ml.machine_memory=126564569088, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  1751. Hot threads at 2020-09-05T21:56:41.030Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1752.  
  1753. 87.0% (434.8ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3373][search][T#23]'
  1754. 8/10 snapshots sharing following 40 elements
  1755. app//com.fasterxml.jackson.core.util.InternCache.intern(InternCache.java:55)
  1756. app//com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer.addName(ByteQuadsCanonicalizer.java:808)
  1757. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.addName(UTF8StreamJsonParser.java:2394)
  1758. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.findName(UTF8StreamJsonParser.java:2278)
  1759. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseLongName(UTF8StreamJsonParser.java:1885)
  1760. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName2(UTF8StreamJsonParser.java:1840)
  1761. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseMediumName(UTF8StreamJsonParser.java:1797)
  1762. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1732)
  1763. app//com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextToken(UTF8StreamJsonParser.java:776)
  1764. app//org.elasticsearch.common.xcontent.json.JsonXContentParser.nextToken(JsonXContentParser.java:52)
  1765. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:337)
  1766. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1767. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  1768. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  1769. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3628/0x000000080162b440.apply(Unknown Source)
  1770. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  1771. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  1772. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  1773. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  1774. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  1775. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  1776. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  1777. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  1778. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  1779. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  1780. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  1781. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  1782. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1783. app//org.elasticsearch.search.SearchService$$Lambda$4731/0x0000000801944040.get(Unknown Source)
  1784. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1785. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1786. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1787. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1788. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1789. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1790. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1791. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1792. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1793. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1794. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1795. 2/10 snapshots sharing following 13 elements
  1796. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  1797. app//org.elasticsearch.search.SearchService$$Lambda$4731/0x0000000801944040.get(Unknown Source)
  1798. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1799. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1800. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1801. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1802. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1803. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1804. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1805. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1806. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1807. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1808. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1809.  
  1810. 78.3% (391.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3373][search][T#35]'
  1811. 2/10 snapshots sharing following 27 elements
  1812. app//org.apache.lucene.search.DisiPriorityQueue.downHeap(DisiPriorityQueue.java:158)
  1813. app//org.apache.lucene.search.DisiPriorityQueue.updateTop(DisiPriorityQueue.java:119)
  1814. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:56)
  1815. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  1816. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  1817. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1818. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1819. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1820. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1821. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1822. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1823. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1824. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1825. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1826. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1827. app//org.elasticsearch.search.SearchService$$Lambda$4626/0x00000008018b7c40.get(Unknown Source)
  1828. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1829. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1830. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1831. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1832. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1833. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1834. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1835. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1836. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1837. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1838. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1839. 3/10 snapshots sharing following 32 elements
  1840. app//org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:40)
  1841. app//org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:194)
  1842. app//org.apache.lucene.search.FilterScorer.score(FilterScorer.java:59)
  1843. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:388)
  1844. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  1845. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  1846. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  1847. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  1848. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  1849. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  1850. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1851. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1852. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1853. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1854. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1855. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1856. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1857. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1858. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1859. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1860. app//org.elasticsearch.search.SearchService$$Lambda$4626/0x00000008018b7c40.get(Unknown Source)
  1861. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1862. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1863. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1864. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1865. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1866. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1867. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1868. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1869. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1870. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1871. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1872. 4/10 snapshots sharing following 24 elements
  1873. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  1874. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  1875. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1876. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1877. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1878. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1879. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1880. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1881. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1882. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1883. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1884. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1885. app//org.elasticsearch.search.SearchService$$Lambda$4626/0x00000008018b7c40.get(Unknown Source)
  1886. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1887. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1888. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1889. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1890. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1891. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1892. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1893. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1894. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1895. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1896. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1897. unique snapshot
  1898. app//org.apache.lucene.search.DisiPriorityQueue.downHeap(DisiPriorityQueue.java:158)
  1899. app//org.apache.lucene.search.DisiPriorityQueue.updateTop(DisiPriorityQueue.java:119)
  1900. app//org.apache.lucene.search.DisjunctionDISIApproximation.advance(DisjunctionDISIApproximation.java:67)
  1901. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:202)
  1902. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  1903. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  1904. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1905. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1906. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1907. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1908. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1909. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1910. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1911. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1912. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1913. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1914. app//org.elasticsearch.search.SearchService$$Lambda$4626/0x00000008018b7c40.get(Unknown Source)
  1915. app//org.elasticsearch.search.SearchService$$Lambda$4628/0x00000008018b7440.get(Unknown Source)
  1916. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1917. app//org.elasticsearch.action.ActionRunnable$$Lambda$4555/0x00000008018a4840.accept(Unknown Source)
  1918. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1919. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1920. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1921. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1922. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1923. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1924. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1925. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1926.  
  1927. ::: {chi1-lapp1520}{RCUByZIpT_Oe8OzNQYl2Dg}{K7eOiyg4S1K9Avbw_uJIBg}{10.128.131.72}{10.128.131.72:9300}{dil}{ml.machine_memory=126564769792, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  1928. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1929.  
  1930. 36.7% (183.5ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1520][search][T#21]'
  1931. 4/10 snapshots sharing following 28 elements
  1932. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  1933. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  1934. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  1935. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  1936. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  1937. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  1938. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1939. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1940. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1941. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1942. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1943. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1944. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1945. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1946. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1947. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1948. app//org.elasticsearch.search.SearchService$$Lambda$4873/0x00000008019c6840.get(Unknown Source)
  1949. app//org.elasticsearch.search.SearchService$$Lambda$4879/0x00000008019c5440.get(Unknown Source)
  1950. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1951. app//org.elasticsearch.action.ActionRunnable$$Lambda$4734/0x00000008019a0c40.accept(Unknown Source)
  1952. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1953. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1954. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1955. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1956. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1957. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1958. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1959. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1960. 2/10 snapshots sharing following 26 elements
  1961. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  1962. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  1963. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  1964. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  1965. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1966. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1967. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1968. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1969. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1970. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1971. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1972. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1973. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  1974. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  1975. app//org.elasticsearch.search.SearchService$$Lambda$4873/0x00000008019c6840.get(Unknown Source)
  1976. app//org.elasticsearch.search.SearchService$$Lambda$4879/0x00000008019c5440.get(Unknown Source)
  1977. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  1978. app//org.elasticsearch.action.ActionRunnable$$Lambda$4734/0x00000008019a0c40.accept(Unknown Source)
  1979. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  1980. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1981. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  1982. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  1983. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  1984. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  1985. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  1986. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  1987. 2/10 snapshots sharing following 25 elements
  1988. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  1989. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  1990. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  1991. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  1992. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  1993. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  1994. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  1995. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  1996. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  1997. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  1998. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  1999. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2000. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2001. app//org.elasticsearch.search.SearchService$$Lambda$4873/0x00000008019c6840.get(Unknown Source)
  2002. app//org.elasticsearch.search.SearchService$$Lambda$4879/0x00000008019c5440.get(Unknown Source)
  2003. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2004. app//org.elasticsearch.action.ActionRunnable$$Lambda$4734/0x00000008019a0c40.accept(Unknown Source)
  2005. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2006. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2007. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2008. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2009. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2010. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2011. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2012. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2013. 2/10 snapshots sharing following 24 elements
  2014. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  2015. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  2016. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2017. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2018. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2019. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2020. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2021. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2022. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2023. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2024. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2025. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2026. app//org.elasticsearch.search.SearchService$$Lambda$4873/0x00000008019c6840.get(Unknown Source)
  2027. app//org.elasticsearch.search.SearchService$$Lambda$4879/0x00000008019c5440.get(Unknown Source)
  2028. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2029. app//org.elasticsearch.action.ActionRunnable$$Lambda$4734/0x00000008019a0c40.accept(Unknown Source)
  2030. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2031. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2032. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2033. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2034. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2035. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2036. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2037. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2038.  
  2039. ::: {chi1-lapp3130}{vxI7XzlORmiJcetP_nDNsw}{ru8hVwRbSqCkSd2Ax_fGjg}{10.128.180.200}{10.128.180.200:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  2040. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2041.  
  2042. 46.6% (233ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3130][transport_worker][T#27]'
  2043. 10/10 snapshots sharing following 76 elements
  2044. java.base@13.0.2/sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method)
  2045. java.base@13.0.2/sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:156)
  2046. java.base@13.0.2/sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:236)
  2047. java.base@13.0.2/sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105)
  2048. java.base@13.0.2/java.nio.file.Files.delete(Files.java:1145)
  2049. app//org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:370)
  2050. app//org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:339)
  2051. app//org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
  2052. app//org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176)
  2053. app//org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
  2054. app//org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:713)
  2055. app//org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:718)
  2056. app//org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38)
  2057. app//org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696)
  2058. app//org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690)
  2059. app//org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:589)
  2060. app//org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:620)
  2061. app//org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:5106)
  2062. app//org.apache.lucene.index.StandardDirectoryReader.lambda$doClose$1(StandardDirectoryReader.java:377)
  2063. app//org.apache.lucene.index.StandardDirectoryReader$$Lambda$305/0x0000000800cd6840.close(Unknown Source)
  2064. app//org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:391)
  2065. app//org.apache.lucene.index.IndexReader.decRef(IndexReader.java:244)
  2066. app//org.apache.lucene.index.IndexReader.close(IndexReader.java:385)
  2067. app//org.apache.lucene.index.FilterDirectoryReader.doClose(FilterDirectoryReader.java:142)
  2068. app//org.apache.lucene.index.IndexReader.decRef(IndexReader.java:244)
  2069. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.decRef(InternalEngine.java:379)
  2070. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.decRef(InternalEngine.java:326)
  2071. app//org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274)
  2072. app//org.elasticsearch.index.engine.Engine.lambda$acquireSearcher$1(Engine.java:683)
  2073. app//org.elasticsearch.index.engine.Engine$$Lambda$4286/0x00000008018a3c40.close(Unknown Source)
  2074. app//org.elasticsearch.index.engine.Engine$Searcher.close(Engine.java:1266)
  2075. app//org.elasticsearch.core.internal.io.IOUtils.close(IOUtils.java:104)
  2076. app//org.elasticsearch.core.internal.io.IOUtils.close(IOUtils.java:86)
  2077. app//org.elasticsearch.common.lease.Releasables.close(Releasables.java:36)
  2078. app//org.elasticsearch.common.lease.Releasables.close(Releasables.java:46)
  2079. app//org.elasticsearch.common.lease.Releasables.close(Releasables.java:51)
  2080. app//org.elasticsearch.search.DefaultSearchContext.doClose(DefaultSearchContext.java:190)
  2081. app//org.elasticsearch.search.internal.SearchContext.closeInternal(SearchContext.java:114)
  2082. app//org.elasticsearch.common.util.concurrent.AbstractRefCounted.decRef(AbstractRefCounted.java:63)
  2083. app//org.elasticsearch.search.internal.SearchContext.close(SearchContext.java:105)
  2084. app//org.elasticsearch.search.SearchService.freeContext(SearchService.java:677)
  2085. app//org.elasticsearch.action.search.SearchTransportService.lambda$registerRequestHandler$2(SearchTransportService.java:295)
  2086. app//org.elasticsearch.action.search.SearchTransportService$$Lambda$2989/0x000000080157bc40.messageReceived(Unknown Source)
  2087. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  2088. app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
  2089. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2090. app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225)
  2091. app//org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:186)
  2092. app//org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:119)
  2093. app//org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:103)
  2094. app//org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:667)
  2095. org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62)
  2096. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
  2097. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
  2098. io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
  2099. io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:326)
  2100. io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:300)
  2101. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
  2102. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
  2103. io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
  2104. io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
  2105. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
  2106. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
  2107. io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
  2108. io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
  2109. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
  2110. io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
  2111. io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
  2112. io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
  2113. io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700)
  2114. io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600)
  2115. io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554)
  2116. io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514)
  2117. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  2118. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  2119. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2120.  
  2121. ::: {chi1-lapp2899}{zH1k3W90Rmeutfe7ca6RGw}{yNHedk5xSlKwfWZOceG3LA}{10.128.179.130}{10.128.179.130:9300}{il}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  2122. Hot threads at 2020-09-05T21:56:41.025Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2123.  
  2124. ::: {chi1-lapp3142}{HfI3p_uJRLO1mSYr7LWasA}{NY9u83A3Rkqx1_1qeRpB-w}{10.128.157.76}{10.128.157.76:9300}{l}{ml.machine_memory=33558077440, ml.max_open_jobs=20, xpack.installed=true}
  2125. Hot threads at 2020-09-05T21:56:41.025Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2126.  
  2127. ::: {chi1-lapp3116}{R-G0S3jFRWiF1qWz9fuoLA}{SCSrqNyKTaSFBQWB5WxmbQ}{10.128.171.133}{10.128.171.133:9300}{lm}{ml.machine_memory=8191987712, ml.max_open_jobs=20, xpack.installed=true}
  2128. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2129.  
  2130. 1.1% (5.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3116][transport_worker][T#4]'
  2131. 5/10 snapshots sharing following 3 elements
  2132. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  2133. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  2134. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2135.  
  2136. ::: {chi1-lapp3136}{CJ7XWpyZRyu2bsfVxK9PLA}{KJSYpcoeSDaNXKP7Np8cXQ}{10.128.175.140}{10.128.175.140:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  2137. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2138.  
  2139. 30.0% (149.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3136][search][T#27]'
  2140. 2/10 snapshots sharing following 22 elements
  2141. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2142. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2143. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2144. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2145. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2146. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2147. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2148. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2149. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2150. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2151. app//org.elasticsearch.search.SearchService$$Lambda$4796/0x0000000801a97040.get(Unknown Source)
  2152. app//org.elasticsearch.search.SearchService$$Lambda$4797/0x0000000801a97440.get(Unknown Source)
  2153. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2154. app//org.elasticsearch.action.ActionRunnable$$Lambda$4359/0x00000008018b1040.accept(Unknown Source)
  2155. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2156. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2157. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2158. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2159. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2160. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2161. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2162. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2163. 8/10 snapshots sharing following 10 elements
  2164. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  2165. java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2166. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
  2167. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  2168. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
  2169. app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
  2170. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
  2171. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  2172. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2173. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2174.  
  2175. 28.3% (141.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3136][search][T#16]'
  2176. 3/10 snapshots sharing following 24 elements
  2177. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  2178. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  2179. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2180. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2181. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2182. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2183. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2184. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2185. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2186. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2187. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2188. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2189. app//org.elasticsearch.search.SearchService$$Lambda$4796/0x0000000801a97040.get(Unknown Source)
  2190. app//org.elasticsearch.search.SearchService$$Lambda$4797/0x0000000801a97440.get(Unknown Source)
  2191. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2192. app//org.elasticsearch.action.ActionRunnable$$Lambda$4359/0x00000008018b1040.accept(Unknown Source)
  2193. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2194. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2195. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2196. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2197. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2198. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2199. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2200. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2201. 7/10 snapshots sharing following 10 elements
  2202. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  2203. java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2204. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
  2205. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  2206. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
  2207. app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
  2208. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
  2209. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  2210. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2211. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2212.  
  2213. ::: {chi1-lapp3121}{0nEWGDv7QZOc03wVGOys7A}{NJJHJyZDR0KzVmynu2GaYw}{10.128.180.199}{10.128.180.199:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  2214. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2215.  
  2216. 100.5% (502.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3121][[allgames_v5][8]: Lucene Merge Thread #125]'
  2217. 3/10 snapshots sharing following 13 elements
  2218. app//org.apache.lucene.index.OrdinalMap$TermsEnumIndex.next(OrdinalMap.java:59)
  2219. app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:255)
  2220. app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
  2221. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:692)
  2222. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2223. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2224. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2225. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2226. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2227. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2228. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2229. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2230. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2231. 7/10 snapshots sharing following 9 elements
  2232. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2233. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2234. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2235. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2236. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2237. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2238. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2239. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2240. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2241.  
  2242. 100.2% (500.8ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3121][[allgames_v8][1]: Lucene Merge Thread #154123]'
  2243. 3/10 snapshots sharing following 18 elements
  2244. app//org.apache.lucene.store.DataInput.readVInt(DataInput.java:126)
  2245. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.next(Lucene80DocValuesProducer.java:969)
  2246. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$TermsDict.seekExact(Lucene80DocValuesProducer.java:987)
  2247. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:909)
  2248. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  2249. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  2250. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:491)
  2251. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  2252. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  2253. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2254. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2255. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2256. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2257. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2258. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2259. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2260. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2261. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2262. 2/10 snapshots sharing following 15 elements
  2263. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:910)
  2264. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  2265. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  2266. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:491)
  2267. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  2268. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  2269. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2270. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2271. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2272. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2273. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2274. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2275. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2276. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2277. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2278. 2/10 snapshots sharing following 13 elements
  2279. app//org.apache.lucene.store.DataOutput.writeVInt(DataOutput.java:191)
  2280. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:503)
  2281. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  2282. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  2283. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2284. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2285. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2286. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2287. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2288. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2289. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2290. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2291. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2292. 2/10 snapshots sharing following 15 elements
  2293. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$BaseSortedSetDocValues.lookupOrd(Lucene80DocValuesProducer.java:909)
  2294. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.lookupOrd(DocValuesConsumer.java:777)
  2295. app//org.apache.lucene.index.SortedSetDocValuesTermsEnum.next(SortedSetDocValuesTermsEnum.java:83)
  2296. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:491)
  2297. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  2298. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  2299. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2300. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2301. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2302. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2303. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2304. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2305. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2306. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2307. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2308. unique snapshot
  2309. java.base@13.0.2/sun.nio.ch.FileDispatcherImpl.write0(Native Method)
  2310. java.base@13.0.2/sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:62)
  2311. java.base@13.0.2/sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:113)
  2312. java.base@13.0.2/sun.nio.ch.IOUtil.write(IOUtil.java:79)
  2313. java.base@13.0.2/sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:280)
  2314. java.base@13.0.2/java.nio.channels.Channels.writeFullyImpl(Channels.java:74)
  2315. java.base@13.0.2/java.nio.channels.Channels.writeFully(Channels.java:97)
  2316. java.base@13.0.2/java.nio.channels.Channels$1.write(Channels.java:172)
  2317. app//org.apache.lucene.store.FSDirectory$FSIndexOutput$1.write(FSDirectory.java:416)
  2318. java.base@13.0.2/java.util.zip.CheckedOutputStream.write(CheckedOutputStream.java:74)
  2319. java.base@13.0.2/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)
  2320. java.base@13.0.2/java.io.BufferedOutputStream.write(BufferedOutputStream.java:127)
  2321. app//org.apache.lucene.store.OutputStreamIndexOutput.writeBytes(OutputStreamIndexOutput.java:53)
  2322. app//org.elasticsearch.common.lucene.store.FilterIndexOutput.writeBytes(FilterIndexOutput.java:59)
  2323. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:73)
  2324. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addTermsDict(Lucene80DocValuesConsumer.java:495)
  2325. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  2326. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  2327. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  2328. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2329. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2330. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2331. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2332. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2333. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2334. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2335. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2336.  
  2337. ::: {chi1-lapp2957}{kNZn5UP5SUK-hWV1wuBXow}{bVtY8balRKeFWg8zJUtlOg}{10.128.130.144}{10.128.130.144:9300}{dil}{ml.machine_memory=126561796096, ml.max_open_jobs=20, xpack.installed=true}
  2338. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2339.  
  2340. ::: {chi1-lapp3143}{Js7aI5s1TH235ikxPiN1eA}{ekMtD0NsQ7OuYEusuUmTVA}{10.128.130.145}{10.128.130.145:9300}{l}{ml.machine_memory=33557970944, ml.max_open_jobs=20, xpack.installed=true}
  2341. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2342.  
  2343. ::: {chi1-lapp3141}{neI_cKr0QtqbpydDWKbxZQ}{nqew_OxySAincUnGr5gGCA}{10.128.133.73}{10.128.133.73:9300}{lm}{ml.machine_memory=8191979520, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  2344. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2345.  
  2346. ::: {chi1-lapp3144}{rCszQSgvQeeSDa3fR5QnlQ}{FPYdyCAfQF2GpoL8igM8hQ}{10.128.131.206}{10.128.131.206:9300}{l}{ml.machine_memory=33557970944, ml.max_open_jobs=20, xpack.installed=true}
  2347. Hot threads at 2020-09-05T21:56:41.026Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2348.  
  2349. ::: {chi1-lapp3131}{fuRR8C-mTFmZlUFZrCtZTA}{C8pGVod0Rh-PzkJEty-Spg}{10.128.169.206}{10.128.169.206:9300}{il}{ml.machine_memory=33558077440, ml.max_open_jobs=20, xpack.installed=true}
  2350. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2351.  
  2352. ::: {chi1-lapp1866}{TJ3wIqE0RHmAO-ULUPDvMA}{dJsChDXUQUGG2Kj1n7cOIg}{10.128.154.12}{10.128.154.12:9300}{dil}{ml.machine_memory=126564630528, ml.max_open_jobs=20, xpack.installed=true}
  2353. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2354.  
  2355. 22.9% (114.5ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1866][[allgames_v8][7]: Lucene Merge Thread #153946]'
  2356. 5/10 snapshots sharing following 22 elements
  2357. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  2358. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  2359. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  2360. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  2361. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  2362. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  2363. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  2364. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  2365. app//org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:155)
  2366. app//org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:168)
  2367. app//org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  2368. app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  2369. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:238)
  2370. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  2371. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:645)
  2372. app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
  2373. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
  2374. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2375. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2376. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2377. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2378. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2379. 5/10 snapshots sharing following 13 elements
  2380. app//org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:168)
  2381. app//org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  2382. app//org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  2383. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:238)
  2384. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  2385. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:645)
  2386. app//org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:229)
  2387. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:106)
  2388. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2389. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2390. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2391. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2392. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2393.  
  2394. 1.2% (5.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1866][search][T#21]'
  2395. 4/10 snapshots sharing following 29 elements
  2396. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  2397. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  2398. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  2399. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3481/0x000000080166f040.apply(Unknown Source)
  2400. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  2401. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  2402. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  2403. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  2404. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  2405. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  2406. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  2407. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  2408. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  2409. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  2410. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  2411. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  2412. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  2413. app//org.elasticsearch.search.SearchService$$Lambda$4816/0x00000008019a8840.get(Unknown Source)
  2414. app//org.elasticsearch.search.SearchService$$Lambda$4790/0x000000080197f440.get(Unknown Source)
  2415. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2416. app//org.elasticsearch.action.ActionRunnable$$Lambda$4395/0x00000008018ba040.accept(Unknown Source)
  2417. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2418. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2419. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2420. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2421. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2422. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2423. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2424. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2425. 6/10 snapshots sharing following 2 elements
  2426. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2427. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2428.  
  2429. ::: {chi1-lapp3122}{G0aL99A-TCq9lBYXdfPzyg}{1mUq-YdbRmiqxcUODYBbOQ}{10.128.169.75}{10.128.169.75:9300}{il}{ml.machine_memory=33558077440, ml.max_open_jobs=20, xpack.installed=true}
  2430. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2431.  
  2432. 14.2% (71ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3122][transport_worker][T#6]'
  2433. 10/10 snapshots sharing following 3 elements
  2434. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  2435. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  2436. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2437.  
  2438. 12.2% (60.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3122][transport_worker][T#7]'
  2439. 5/10 snapshots sharing following 3 elements
  2440. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  2441. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  2442. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2443.  
  2444. ::: {chi1-lapp1863}{XTBdcq2oTcG3nTeK3kE7hg}{GIa5ZULBR32y-3sDIXGgGA}{10.128.131.9}{10.128.131.9:9300}{dil}{ml.machine_memory=126564630528, ml.max_open_jobs=20, xpack.installed=true}
  2445. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2446.  
  2447. 59.1% (295.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1863][search][T#38]'
  2448. 5/10 snapshots sharing following 29 elements
  2449. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  2450. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readValue(AbstractXContentParser.java:375)
  2451. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.lambda$readMap$0(AbstractXContentParser.java:322)
  2452. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser$$Lambda$3481/0x000000080165cc40.apply(Unknown Source)
  2453. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readGenericMap(AbstractXContentParser.java:342)
  2454. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:322)
  2455. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.readMap(AbstractXContentParser.java:302)
  2456. app//org.elasticsearch.common.xcontent.support.AbstractXContentParser.map(AbstractXContentParser.java:266)
  2457. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:145)
  2458. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:115)
  2459. app//org.elasticsearch.common.xcontent.XContentHelper.convertToMap(XContentHelper.java:91)
  2460. app//org.elasticsearch.search.lookup.SourceLookup.sourceAsMapAndType(SourceLookup.java:86)
  2461. app//org.elasticsearch.search.lookup.SourceLookup.loadSourceIfNeeded(SourceLookup.java:62)
  2462. app//org.elasticsearch.search.lookup.SourceLookup.filter(SourceLookup.java:132)
  2463. app//org.elasticsearch.search.fetch.subphase.FetchSourcePhase.hitExecute(FetchSourcePhase.java:62)
  2464. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:170)
  2465. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  2466. app//org.elasticsearch.search.SearchService$$Lambda$4682/0x0000000801996040.get(Unknown Source)
  2467. app//org.elasticsearch.search.SearchService$$Lambda$4618/0x0000000801947440.get(Unknown Source)
  2468. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2469. app//org.elasticsearch.action.ActionRunnable$$Lambda$4533/0x0000000801925840.accept(Unknown Source)
  2470. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2471. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2472. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2473. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2474. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2475. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2476. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2477. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2478. 2/10 snapshots sharing following 24 elements
  2479. app//org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:138)
  2480. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState$1.fillBuffer(CompressingStoredFieldsReader.java:526)
  2481. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState$1.readBytes(CompressingStoredFieldsReader.java:545)
  2482. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.readField(CompressingStoredFieldsReader.java:210)
  2483. app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.visitDocument(CompressingStoredFieldsReader.java:590)
  2484. app//org.apache.lucene.index.CodecReader.document(CodecReader.java:84)
  2485. app//org.apache.lucene.index.FilterLeafReader.document(FilterLeafReader.java:355)
  2486. app//org.elasticsearch.search.fetch.FetchPhase.loadStoredFields(FetchPhase.java:426)
  2487. app//org.elasticsearch.search.fetch.FetchPhase.getSearchFields(FetchPhase.java:233)
  2488. app//org.elasticsearch.search.fetch.FetchPhase.createSearchHit(FetchPhase.java:215)
  2489. app//org.elasticsearch.search.fetch.FetchPhase.execute(FetchPhase.java:163)
  2490. app//org.elasticsearch.search.SearchService.lambda$executeFetchPhase$5(SearchService.java:507)
  2491. app//org.elasticsearch.search.SearchService$$Lambda$4682/0x0000000801996040.get(Unknown Source)
  2492. app//org.elasticsearch.search.SearchService$$Lambda$4618/0x0000000801947440.get(Unknown Source)
  2493. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2494. app//org.elasticsearch.action.ActionRunnable$$Lambda$4533/0x0000000801925840.accept(Unknown Source)
  2495. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2496. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2497. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2498. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2499. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2500. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2501. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2502. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2503.  
  2504. ::: {chi1-lapp2897}{1H40RMYaSO6C1cDKvquimw}{0cbAxJVfQkCOHeVyRURTnA}{10.128.179.2}{10.128.179.2:9300}{dil}{ml.machine_memory=126564573184, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  2505. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2506.  
  2507. 33.3% (166.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2897][search][T#20]'
  2508. 3/10 snapshots sharing following 24 elements
  2509. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  2510. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  2511. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2512. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2513. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2514. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2515. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2516. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2517. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2518. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2519. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2520. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2521. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018afc40.get(Unknown Source)
  2522. app//org.elasticsearch.search.SearchService$$Lambda$4581/0x00000008018af040.get(Unknown Source)
  2523. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2524. app//org.elasticsearch.action.ActionRunnable$$Lambda$4491/0x000000080188c840.accept(Unknown Source)
  2525. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2526. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2527. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2528. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2529. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2530. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2531. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2532. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2533. 2/10 snapshots sharing following 31 elements
  2534. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  2535. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  2536. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  2537. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  2538. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  2539. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  2540. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  2541. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  2542. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  2543. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2544. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2545. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2546. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2547. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2548. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2549. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2550. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2551. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2552. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2553. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018afc40.get(Unknown Source)
  2554. app//org.elasticsearch.search.SearchService$$Lambda$4581/0x00000008018af040.get(Unknown Source)
  2555. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2556. app//org.elasticsearch.action.ActionRunnable$$Lambda$4491/0x000000080188c840.accept(Unknown Source)
  2557. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2558. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2559. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2560. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2561. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2562. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2563. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2564. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2565. 5/10 snapshots sharing following 2 elements
  2566. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2567. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2568.  
  2569. ::: {chi1-lapp3139}{KQc9ctXwTRqa-pmEddAcpQ}{McRMNlMtQPCzfhVtkL6iGg}{10.128.150.73}{10.128.150.73:9300}{il}{ml.machine_memory=33558077440, ml.max_open_jobs=20, xpack.installed=true}
  2570. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2571.  
  2572. 8.7% (43.7ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3139][transport_worker][T#6]'
  2573. 2/10 snapshots sharing following 3 elements
  2574. io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
  2575. io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
  2576. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2577.  
  2578. ::: {chi1-lapp3117}{zu2Ky3xrTVygPygnDiBw9A}{s2A8g--3SCWwl6VgAlw4ZA}{10.128.173.210}{10.128.173.210:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  2579. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2580.  
  2581. 100.1% (500.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3117][[allgames_v5][0]: Lucene Merge Thread #154997]'
  2582. 2/10 snapshots sharing following 17 elements
  2583. app//org.apache.lucene.util.fst.Builder.compileNode(Builder.java:253)
  2584. app//org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:349)
  2585. app//org.apache.lucene.util.fst.Builder.add(Builder.java:453)
  2586. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.append(BlockTreeTermsWriter.java:502)
  2587. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$PendingBlock.compileIndex(BlockTreeTermsWriter.java:474)
  2588. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.writeBlocks(BlockTreeTermsWriter.java:634)
  2589. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.finish(BlockTreeTermsWriter.java:935)
  2590. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:347)
  2591. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2592. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2593. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2594. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2595. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2596. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2597. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2598. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2599. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2600. 2/10 snapshots sharing following 11 elements
  2601. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:865)
  2602. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2603. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2604. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2605. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2606. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2607. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2608. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2609. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2610. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2611. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2612. 2/10 snapshots sharing following 12 elements
  2613. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  2614. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  2615. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2616. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2617. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2618. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2619. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2620. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2621. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2622. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2623. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2624. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2625. 4/10 snapshots sharing following 8 elements
  2626. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  2627. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  2628. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  2629. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2630. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2631. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2632. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2633. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2634.  
  2635. 10.4% (51.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3117][refresh][T#3]'
  2636. 2/10 snapshots sharing following 30 elements
  2637. app//org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:263)
  2638. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:138)
  2639. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  2640. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  2641. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  2642. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  2643. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  2644. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  2645. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  2646. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  2647. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  2648. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  2649. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  2650. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2651. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  2652. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  2653. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  2654. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2655. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  2656. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  2657. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  2658. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  2659. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  2660. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  2661. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  2662. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  2663. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  2664. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2665. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2666. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2667. 3/10 snapshots sharing following 36 elements
  2668. app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:128)
  2669. app//org.apache.lucene.util.MSBRadixSorter.radixSort(MSBRadixSorter.java:174)
  2670. app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:128)
  2671. app//org.apache.lucene.util.MSBRadixSorter.sort(MSBRadixSorter.java:121)
  2672. app//org.apache.lucene.util.BytesRefHash.sort(BytesRefHash.java:185)
  2673. app//org.apache.lucene.index.TermsHashPerField.sortPostings(TermsHashPerField.java:97)
  2674. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:79)
  2675. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  2676. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  2677. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  2678. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  2679. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  2680. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  2681. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  2682. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  2683. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  2684. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  2685. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  2686. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  2687. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2688. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  2689. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  2690. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  2691. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2692. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  2693. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  2694. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  2695. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  2696. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  2697. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  2698. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  2699. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  2700. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  2701. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2702. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2703. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2704. 3/10 snapshots sharing following 34 elements
  2705. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  2706. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  2707. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2708. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  2709. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  2710. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  2711. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  2712. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  2713. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  2714. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  2715. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  2716. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  2717. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  2718. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  2719. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  2720. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  2721. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  2722. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2723. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  2724. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  2725. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  2726. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2727. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  2728. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  2729. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  2730. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  2731. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  2732. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  2733. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  2734. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  2735. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  2736. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2737. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2738. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2739. 2/10 snapshots sharing following 32 elements
  2740. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2741. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  2742. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  2743. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  2744. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  2745. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  2746. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  2747. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  2748. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  2749. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  2750. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  2751. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  2752. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  2753. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  2754. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  2755. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2756. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  2757. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  2758. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  2759. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  2760. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  2761. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  2762. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  2763. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  2764. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  2765. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  2766. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  2767. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  2768. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  2769. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2770. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2771. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2772.  
  2773. ::: {chi1-lapp2953}{DUSyMZjOQRqCQaD-XGb8aw}{AWlqUeOwR2er2N9QLXk9xw}{10.128.135.134}{10.128.135.134:9300}{il}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  2774. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2775.  
  2776. ::: {chi1-lapp2665}{q1CGzH_EQlqKJ4xPel2zjA}{XzbB4wVJTVusfnSJwNOpSw}{10.128.154.19}{10.128.154.19:9300}{dil}{ml.machine_memory=126564622336, ml.max_open_jobs=20, xpack.installed=true}
  2777. Hot threads at 2020-09-05T21:56:41.027Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2778.  
  2779. 41.4% (206.9ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2665][write][T#27]'
  2780. 3/10 snapshots sharing following 40 elements
  2781. app//org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:250)
  2782. app//org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:495)
  2783. app//org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
  2784. app//org.apache.lucene.index.IndexWriter.softUpdateDocument(IndexWriter.java:1654)
  2785. app//org.elasticsearch.index.engine.InternalEngine.updateDocs(InternalEngine.java:1259)
  2786. app//org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1090)
  2787. app//org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:934)
  2788. app//org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:813)
  2789. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:785)
  2790. app//org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:742)
  2791. app//org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:267)
  2792. app//org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:157)
  2793. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2794. app//org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:202)
  2795. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114)
  2796. app//org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:81)
  2797. app//org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:895)
  2798. app//org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:109)
  2799. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:374)
  2800. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:297)
  2801. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction$$Lambda$4608/0x0000000801978c40.accept(Unknown Source)
  2802. app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
  2803. app//org.elasticsearch.index.shard.IndexShard.lambda$wrapPrimaryOperationPermitListener$24(IndexShard.java:2791)
  2804. app//org.elasticsearch.index.shard.IndexShard$$Lambda$4597/0x000000080196b040.accept(Unknown Source)
  2805. app//org.elasticsearch.action.ActionListener$3.onResponse(ActionListener.java:113)
  2806. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:285)
  2807. app//org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:237)
  2808. app//org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2765)
  2809. app//org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:836)
  2810. app//org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:293)
  2811. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2812. app//org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:256)
  2813. app//org.elasticsearch.action.support.replication.TransportReplicationAction$$Lambda$2875/0x000000080151b440.messageReceived(Unknown Source)
  2814. app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
  2815. app//org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:750)
  2816. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2817. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2818. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2819. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2820. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2821. 7/10 snapshots sharing following 10 elements
  2822. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  2823. java.base@13.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
  2824. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:743)
  2825. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  2826. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1366)
  2827. app//org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:165)
  2828. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
  2829. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  2830. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2831. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2832.  
  2833. ::: {chi1-lapp3138}{Cyj9Xo-vTWiM15eOxhELkg}{z-BJEgFLTeS8WVfP1UB69g}{10.128.177.73}{10.128.177.73:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  2834. Hot threads at 2020-09-05T21:56:41.029Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2835.  
  2836. 93.9% (469.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3138][[allgames_v8][5]: Lucene Merge Thread #159161]'
  2837. 2/10 snapshots sharing following 14 elements
  2838. app//org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:359)
  2839. app//org.apache.lucene.index.MappedMultiFields$MappedMultiTermsEnum.postings(MappedMultiFields.java:127)
  2840. app//org.apache.lucene.codecs.PushPostingsWriterBase.writeTerm(PushPostingsWriterBase.java:129)
  2841. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:865)
  2842. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2843. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2844. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2845. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2846. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2847. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2848. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2849. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2850. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2851. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2852. 4/10 snapshots sharing following 12 elements
  2853. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.pushTerm(BlockTreeTermsWriter.java:906)
  2854. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  2855. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2856. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2857. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2858. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2859. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2860. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2861. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2862. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2863. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2864. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2865. 2/10 snapshots sharing following 11 elements
  2866. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter$TermsWriter.write(BlockTreeTermsWriter.java:870)
  2867. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  2868. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2869. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2870. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2871. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2872. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2873. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2874. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2875. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2876. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2877. 2/10 snapshots sharing following 14 elements
  2878. app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
  2879. app//org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  2880. app//org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2881. app//org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
  2882. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:336)
  2883. app//org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
  2884. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.merge(PerFieldPostingsFormat.java:197)
  2885. app//org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:245)
  2886. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:140)
  2887. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  2888. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  2889. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  2890. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  2891. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  2892.  
  2893. 32.5% (162.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3138][search][T#21]'
  2894. 2/10 snapshots sharing following 34 elements
  2895. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:107)
  2896. app//org.elasticsearch.search.lookup.LeafDocLookup.get(LeafDocLookup.java:41)
  2897. org.elasticsearch.painless.PainlessScript$Script.execute(double titlePopCount = doc['popularWordCountInTitle'].empty ? 0 : (double)doc['popularWordCountInTitle'].value; double desPopCount = doc['popularWordCountInDescription'].empty ? 0 : (double)doc['popularWordCountInDescription'].value; if (((3*titlePopCount) ...:28)
  2898. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  2899. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  2900. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  2901. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  2902. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  2903. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  2904. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  2905. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  2906. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  2907. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2908. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2909. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2910. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2911. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2912. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2913. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2914. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2915. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2916. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2917. app//org.elasticsearch.search.SearchService$$Lambda$4861/0x0000000801aa7040.get(Unknown Source)
  2918. app//org.elasticsearch.search.SearchService$$Lambda$4862/0x0000000801aa7440.get(Unknown Source)
  2919. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2920. app//org.elasticsearch.action.ActionRunnable$$Lambda$4435/0x00000008018e3040.accept(Unknown Source)
  2921. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2922. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2923. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2924. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2925. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2926. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2927. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2928. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2929. 2/10 snapshots sharing following 25 elements
  2930. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  2931. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  2932. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  2933. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2934. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2935. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2936. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2937. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2938. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2939. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2940. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2941. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2942. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2943. app//org.elasticsearch.search.SearchService$$Lambda$4861/0x0000000801aa7040.get(Unknown Source)
  2944. app//org.elasticsearch.search.SearchService$$Lambda$4862/0x0000000801aa7440.get(Unknown Source)
  2945. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2946. app//org.elasticsearch.action.ActionRunnable$$Lambda$4435/0x00000008018e3040.accept(Unknown Source)
  2947. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2948. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2949. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2950. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2951. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2952. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2953. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2954. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2955. 2/10 snapshots sharing following 31 elements
  2956. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  2957. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  2958. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  2959. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  2960. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  2961. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  2962. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  2963. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  2964. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  2965. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  2966. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  2967. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  2968. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  2969. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  2970. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  2971. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  2972. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  2973. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  2974. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  2975. app//org.elasticsearch.search.SearchService$$Lambda$4861/0x0000000801aa7040.get(Unknown Source)
  2976. app//org.elasticsearch.search.SearchService$$Lambda$4862/0x0000000801aa7440.get(Unknown Source)
  2977. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  2978. app//org.elasticsearch.action.ActionRunnable$$Lambda$4435/0x00000008018e3040.accept(Unknown Source)
  2979. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  2980. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2981. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  2982. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  2983. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  2984. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  2985. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2986. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2987. 4/10 snapshots sharing following 2 elements
  2988. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  2989. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  2990.  
  2991. ::: {chi1-lapp2670}{-aGx_vxgQ3mMSFUtzDighg}{m4Vx_6S2RPOXnQLxb_TrWQ}{10.128.166.11}{10.128.166.11:9300}{dil}{ml.machine_memory=126564622336, ml.max_open_jobs=20, xpack.installed=true}
  2992. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2993.  
  2994. 88.9% (444.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2670][[allgames_v5][9]: Lucene Merge Thread #120]'
  2995. 3/10 snapshots sharing following 13 elements
  2996. app//org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:202)
  2997. app//org.apache.lucene.index.OrdinalMap.<init>(OrdinalMap.java:261)
  2998. app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
  2999. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:692)
  3000. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  3001. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  3002. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  3003. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  3004. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  3005. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3006. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3007. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3008. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3009. 2/10 snapshots sharing following 11 elements
  3010. app//org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
  3011. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:692)
  3012. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  3013. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  3014. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  3015. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  3016. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  3017. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3018. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3019. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3020. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3021. 3/10 snapshots sharing following 13 elements
  3022. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$24.nextOrd(Lucene80DocValuesProducer.java:1357)
  3023. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.nextOrd(DocValuesConsumer.java:761)
  3024. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:608)
  3025. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  3026. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  3027. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  3028. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  3029. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  3030. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  3031. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3032. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3033. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3034. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3035. 2/10 snapshots sharing following 15 elements
  3036. app//org.apache.lucene.util.packed.MonotonicLongValues.get(MonotonicLongValues.java:40)
  3037. app//org.apache.lucene.util.packed.PackedLongValues.get(PackedLongValues.java:108)
  3038. app//org.apache.lucene.index.OrdinalMap$3.get(OrdinalMap.java:312)
  3039. app//org.apache.lucene.codecs.DocValuesConsumer$5$1.nextOrd(DocValuesConsumer.java:765)
  3040. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:608)
  3041. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  3042. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  3043. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  3044. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  3045. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  3046. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  3047. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3048. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3049. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3050. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3051.  
  3052. 18.3% (91.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2670][refresh][T#3]'
  3053. 2/10 snapshots sharing following 32 elements
  3054. app//org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:344)
  3055. app//org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:170)
  3056. app//org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:97)
  3057. app//org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:177)
  3058. app//org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:468)
  3059. app//org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:555)
  3060. app//org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:722)
  3061. app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:494)
  3062. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  3063. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  3064. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  3065. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  3066. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  3067. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  3068. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  3069. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  3070. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  3071. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  3072. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  3073. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  3074. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  3075. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  3076. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  3077. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  3078. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  3079. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  3080. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  3081. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  3082. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  3083. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3084. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3085. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3086. 2/10 snapshots sharing following 24 elements
  3087. app//org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:297)
  3088. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:272)
  3089. app//org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:262)
  3090. app//org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:112)
  3091. app//org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:165)
  3092. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:66)
  3093. app//org.elasticsearch.index.engine.ElasticsearchReaderManager.refreshIfNeeded(ElasticsearchReaderManager.java:40)
  3094. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  3095. app//org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  3096. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:344)
  3097. app//org.elasticsearch.index.engine.InternalEngine$ExternalReaderManager.refreshIfNeeded(InternalEngine.java:326)
  3098. app//org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  3099. app//org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
  3100. app//org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1611)
  3101. app//org.elasticsearch.index.engine.InternalEngine.maybeRefresh(InternalEngine.java:1590)
  3102. app//org.elasticsearch.index.shard.IndexShard.scheduledRefresh(IndexShard.java:3236)
  3103. app//org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:827)
  3104. app//org.elasticsearch.index.IndexService.access$200(IndexService.java:108)
  3105. app//org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:959)
  3106. app//org.elasticsearch.common.util.concurrent.AbstractAsyncTask.run(AbstractAsyncTask.java:144)
  3107. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633)
  3108. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3109. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3110. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3111. 6/10 snapshots sharing following 9 elements
  3112. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  3113. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  3114. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
  3115. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
  3116. java.base@13.0.2/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
  3117. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
  3118. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
  3119. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3120. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3121.  
  3122. 1.1% (5.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2670][[allgames_v8][2]: Lucene Merge Thread #148513]'
  3123. 10/10 snapshots sharing following 16 elements
  3124. java.base@13.0.2/jdk.internal.misc.Unsafe.park(Native Method)
  3125. java.base@13.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:235)
  3126. java.base@13.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
  3127. app//org.apache.lucene.index.MergePolicy$OneMergeProgress.pauseNanos(MergePolicy.java:156)
  3128. app//org.apache.lucene.index.MergeRateLimiter.maybePause(MergeRateLimiter.java:148)
  3129. app//org.apache.lucene.index.MergeRateLimiter.pause(MergeRateLimiter.java:93)
  3130. app//org.apache.lucene.store.RateLimitedIndexOutput.checkRate(RateLimitedIndexOutput.java:78)
  3131. app//org.apache.lucene.store.RateLimitedIndexOutput.writeBytes(RateLimitedIndexOutput.java:72)
  3132. app//org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  3133. app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.write(Lucene50CompoundFormat.java:96)
  3134. app//org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:5014)
  3135. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4527)
  3136. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3137. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3138. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3139. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3140.  
  3141. ::: {chi1-lapp3369}{W3q97kxSTceBzTPezyW2Aw}{ZoF0WGxeQuOQXG1hzp1BdA}{10.128.166.131}{10.128.166.131:9300}{dil}{ml.machine_memory=126564573184, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  3142. Hot threads at 2020-09-05T21:56:41.074Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3143.  
  3144. 86.3% (431.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3369][search][T#28]'
  3145. 4/10 snapshots sharing following 24 elements
  3146. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3147. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3148. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3149. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3150. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3151. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3152. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3153. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3154. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3155. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3156. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3157. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3158. app//org.elasticsearch.search.SearchService$$Lambda$4579/0x00000008018af040.get(Unknown Source)
  3159. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018af440.get(Unknown Source)
  3160. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3161. app//org.elasticsearch.action.ActionRunnable$$Lambda$4516/0x0000000801893c40.accept(Unknown Source)
  3162. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3163. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3164. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3165. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3166. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3167. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3168. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3169. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3170. 5/10 snapshots sharing following 31 elements
  3171. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3172. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3173. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3174. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3175. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3176. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3177. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3178. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3179. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3180. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3181. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3182. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3183. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3184. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3185. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3186. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3187. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3188. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3189. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3190. app//org.elasticsearch.search.SearchService$$Lambda$4579/0x00000008018af040.get(Unknown Source)
  3191. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018af440.get(Unknown Source)
  3192. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3193. app//org.elasticsearch.action.ActionRunnable$$Lambda$4516/0x0000000801893c40.accept(Unknown Source)
  3194. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3195. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3196. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3197. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3198. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3199. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3200. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3201. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3202. unique snapshot
  3203. app//org.apache.lucene.index.FilterNumericDocValues.advanceExact(FilterNumericDocValues.java:54)
  3204. app//org.apache.lucene.search.FieldComparator$DoubleComparator.getValueForDoc(FieldComparator.java:181)
  3205. app//org.apache.lucene.search.FieldComparator$DoubleComparator.compareBottom(FieldComparator.java:195)
  3206. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:50)
  3207. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3208. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3209. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3210. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3211. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3212. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3213. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3214. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3215. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3216. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3217. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3218. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3219. app//org.elasticsearch.search.SearchService$$Lambda$4579/0x00000008018af040.get(Unknown Source)
  3220. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018af440.get(Unknown Source)
  3221. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3222. app//org.elasticsearch.action.ActionRunnable$$Lambda$4516/0x0000000801893c40.accept(Unknown Source)
  3223. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3224. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3225. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3226. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3227. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3228. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3229. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3230. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3231.  
  3232. 69.9% (349.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3369][search][T#43]'
  3233. 3/10 snapshots sharing following 31 elements
  3234. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3235. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3236. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3237. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3238. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3239. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3240. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3241. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3242. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3243. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3244. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3245. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3246. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3247. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3248. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3249. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3250. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3251. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3252. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3253. app//org.elasticsearch.search.SearchService$$Lambda$4579/0x00000008018af040.get(Unknown Source)
  3254. app//org.elasticsearch.search.SearchService$$Lambda$4580/0x00000008018af440.get(Unknown Source)
  3255. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3256. app//org.elasticsearch.action.ActionRunnable$$Lambda$4516/0x0000000801893c40.accept(Unknown Source)
  3257. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3258. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3259. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3260. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3261. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3262. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3263. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3264. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3265. 7/10 snapshots sharing following 2 elements
  3266. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3267. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3268.  
  3269. ::: {chi1-lapp2954}{LpyCqszcQZ-XpWK2sGFviQ}{I1G8PZMVSJ2kGVbcl8YXyA}{10.128.132.139}{10.128.132.139:9300}{il}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  3270. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3271.  
  3272. ::: {chi1-lapp3123}{hH06vQngTg-i-s_3Hia5WQ}{AturzUwWRLm88f5eyS_NHw}{10.128.131.12}{10.128.131.12:9300}{lm}{ml.machine_memory=8191987712, ml.max_open_jobs=20, xpack.installed=true}
  3273. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3274.  
  3275. ::: {chi1-lapp2973}{4W33DyEeSxOe9gu3XfGG6A}{xkJCF2PBTNqddp0cP_vCbQ}{10.128.129.67}{10.128.129.67:9300}{dil}{ml.machine_memory=126562263040, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  3276. Hot threads at 2020-09-05T21:56:41.032Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3277.  
  3278. 71.5% (357.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2973][search][T#6]'
  3279. 2/10 snapshots sharing following 25 elements
  3280. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  3281. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3282. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3283. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3284. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3285. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3286. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3287. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3288. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3289. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3290. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3291. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3292. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3293. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3294. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3295. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3296. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3297. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3298. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3299. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3300. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3301. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3302. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3303. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3304. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3305.  
  3306. 69.6% (348.2ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2973][search][T#25]'
  3307. 3/10 snapshots sharing following 26 elements
  3308. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  3309. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  3310. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3311. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3312. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3313. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3314. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3315. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3316. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3317. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3318. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3319. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3320. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3321. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3322. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3323. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3324. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3325. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3326. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3327. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3328. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3329. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3330. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3331. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3332. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3333. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3334. 3/10 snapshots sharing following 31 elements
  3335. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3336. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3337. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3338. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3339. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3340. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3341. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3342. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3343. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3344. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3345. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3346. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3347. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3348. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3349. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3350. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3351. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3352. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3353. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3354. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3355. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3356. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3357. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3358. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3359. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3360. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3361. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3362. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3363. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3364. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3365. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3366. 2/10 snapshots sharing following 26 elements
  3367. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3368. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3369. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3370. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3371. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3372. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3373. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3374. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3375. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3376. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3377. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3378. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3379. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3380. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3381. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3382. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3383. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3384. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3385. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3386. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3387. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3388. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3389. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3390. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3391. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3392. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3393. 2/10 snapshots sharing following 24 elements
  3394. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3395. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3396. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3397. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3398. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3399. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3400. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3401. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3402. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3403. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3404. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3405. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3406. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3407. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3408. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3409. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3410. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3411. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3412. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3413. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3414. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3415. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3416. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3417. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3418.  
  3419. 61.1% (305.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp2973][search][T#34]'
  3420. 2/10 snapshots sharing following 28 elements
  3421. app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.decodeAndPrefixSum(ForDeltaUtil.java:79)
  3422. app//org.apache.lucene.codecs.lucene84.Lucene84PostingsReader$BlockDocsEnum.refillDocs(Lucene84PostingsReader.java:402)
  3423. app//org.apache.lucene.codecs.lucene84.Lucene84PostingsReader$BlockDocsEnum.advance(Lucene84PostingsReader.java:486)
  3424. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  3425. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3426. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3427. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3428. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3429. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3430. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3431. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3432. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3433. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3434. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3435. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3436. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3437. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3438. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3439. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3440. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3441. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3442. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3443. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3444. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3445. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3446. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3447. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3448. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3449. 2/10 snapshots sharing following 28 elements
  3450. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3451. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3452. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3453. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3454. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3455. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3456. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3457. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3458. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3459. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3460. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3461. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3462. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3463. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3464. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3465. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3466. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3467. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3468. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3469. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3470. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3471. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3472. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3473. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3474. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3475. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3476. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3477. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3478. 3/10 snapshots sharing following 25 elements
  3479. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:55)
  3480. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3481. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3482. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3483. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3484. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3485. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3486. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3487. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3488. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3489. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3490. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3491. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3492. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3493. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3494. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3495. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3496. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3497. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3498. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3499. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3500. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3501. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3502. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3503. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3504. 2/10 snapshots sharing following 27 elements
  3505. app//org.apache.lucene.search.DisiPriorityQueue.downHeap(DisiPriorityQueue.java:158)
  3506. app//org.apache.lucene.search.DisiPriorityQueue.updateTop(DisiPriorityQueue.java:119)
  3507. app//org.apache.lucene.search.DisjunctionDISIApproximation.nextDoc(DisjunctionDISIApproximation.java:56)
  3508. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3509. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3510. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3511. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3512. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3513. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3514. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3515. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3516. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3517. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3518. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3519. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3520. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3521. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3522. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3523. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3524. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3525. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3526. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3527. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3528. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3529. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3530. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3531. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3532. unique snapshot
  3533. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer$SparseNumericDocValues.advanceExact(Lucene80DocValuesProducer.java:424)
  3534. app//org.apache.lucene.index.FilterNumericDocValues.advanceExact(FilterNumericDocValues.java:54)
  3535. app//org.apache.lucene.search.FieldComparator$DoubleComparator.getValueForDoc(FieldComparator.java:181)
  3536. app//org.apache.lucene.search.FieldComparator$DoubleComparator.compareBottom(FieldComparator.java:195)
  3537. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:50)
  3538. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3539. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3540. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3541. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3542. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3543. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3544. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3545. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3546. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3547. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3548. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3549. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3550. app//org.elasticsearch.search.SearchService$$Lambda$4760/0x00000008018d5c40.get(Unknown Source)
  3551. app//org.elasticsearch.search.SearchService$$Lambda$4767/0x00000008018e0040.get(Unknown Source)
  3552. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3553. app//org.elasticsearch.action.ActionRunnable$$Lambda$4550/0x0000000801899440.accept(Unknown Source)
  3554. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3555. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3556. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3557. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3558. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3559. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3560. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3561. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3562.  
  3563. ::: {chi1-lapp3126}{5sdq8ZyZRyahKuuUq6kL5Q}{SPq6fUNKS4y-VGYbSoL_bA}{10.128.178.134}{10.128.178.134:9300}{dil}{ml.machine_memory=126564618240, ml.max_open_jobs=20, xpack.installed=true}
  3564. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3565.  
  3566. ::: {chi1-lapp1860}{eGeK_OEyQHyfenBSkQeIZg}{SeeA5c04SnOtc45f4jZR5w}{10.128.148.133}{10.128.148.133:9300}{dil}{ml.machine_memory=126564630528, ml.max_open_jobs=20, xpack.installed=true}
  3567. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3568.  
  3569. 38.3% (191.6ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp1860][[allgames_v8][9]: Lucene Merge Thread #45833]'
  3570. 10/10 snapshots sharing following 11 elements
  3571. app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesConsumer.addSortedSetField(Lucene80DocValuesConsumer.java:675)
  3572. app//org.apache.lucene.codecs.DocValuesConsumer.mergeSortedSetField(DocValuesConsumer.java:695)
  3573. app//org.apache.lucene.codecs.DocValuesConsumer.merge(DocValuesConsumer.java:141)
  3574. app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.merge(PerFieldDocValuesFormat.java:152)
  3575. app//org.apache.lucene.index.SegmentMerger.mergeDocValues(SegmentMerger.java:196)
  3576. app//org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:151)
  3577. app//org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4463)
  3578. app//org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4057)
  3579. app//org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  3580. app//org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  3581. app//org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  3582.  
  3583. ::: {chi1-lapp3129}{dnrhnwE3SQO1gIhTO7nH7Q}{4tw2U8PgQxaruVGxqb8p3w}{10.128.131.82}{10.128.131.82:9300}{lm}{ml.machine_memory=8191979520, ml.max_open_jobs=20, xpack.installed=true}
  3584. Hot threads at 2020-09-05T21:56:41.028Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3585.  
  3586. ::: {chi1-lapp3220}{eqXqiu3pQ3qV67CHItMVNg}{va6hl85JTOeTe3hPw-XV3w}{10.128.177.136}{10.128.177.136:9300}{dil}{ml.machine_memory=126564614144, ml.max_open_jobs=20, xpack.installed=true}
  3587. Hot threads at 2020-09-05T21:56:41.029Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3588.  
  3589. ::: {chi1-lapp3394}{hN_skQdzRJOa6fr2_B3hrg}{ZqaSopXgQguMvqnD8wV1rg}{10.128.167.4}{10.128.167.4:9300}{dil}{ml.machine_memory=126564573184, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  3590. Hot threads at 2020-09-05T21:56:41.012Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3591.  
  3592. 77.8% (389ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3394][search][T#6]'
  3593. 5/10 snapshots sharing following 31 elements
  3594. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3595. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3596. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3597. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3598. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3599. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3600. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3601. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3602. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3603. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3604. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3605. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3606. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3607. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3608. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3609. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3610. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3611. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3612. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3613. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3614. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3615. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3616. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3617. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3618. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3619. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3620. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3621. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3622. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3623. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3624. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3625. 2/10 snapshots sharing following 39 elements
  3626. java.base@13.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:796)
  3627. app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179)
  3628. app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
  3629. app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54)
  3630. app//org.apache.lucene.store.DataInput.readInt(DataInput.java:103)
  3631. app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183)
  3632. app//org.apache.lucene.store.DataInput.readLong(DataInput.java:158)
  3633. app//org.apache.lucene.store.BufferedIndexInput.readLong(BufferedIndexInput.java:196)
  3634. app//org.apache.lucene.store.DataInput.readLELongs(DataInput.java:177)
  3635. app//org.apache.lucene.codecs.lucene84.ForUtil.decode4(ForUtil.java:641)
  3636. app//org.apache.lucene.codecs.lucene84.ForUtil.decodeAndPrefixSum(ForUtil.java:516)
  3637. app//org.apache.lucene.codecs.lucene84.ForDeltaUtil.decodeAndPrefixSum(ForDeltaUtil.java:79)
  3638. app//org.apache.lucene.codecs.lucene84.Lucene84PostingsReader$BlockDocsEnum.refillDocs(Lucene84PostingsReader.java:402)
  3639. app//org.apache.lucene.codecs.lucene84.Lucene84PostingsReader$BlockDocsEnum.advance(Lucene84PostingsReader.java:486)
  3640. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  3641. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3642. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3643. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3644. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3645. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3646. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3647. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3648. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3649. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3650. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3651. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3652. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3653. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3654. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3655. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3656. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3657. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3658. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3659. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3660. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3661. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3662. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3663. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3664. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3665. 3/10 snapshots sharing following 24 elements
  3666. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3667. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3668. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3669. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3670. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3671. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3672. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3673. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3674. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3675. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3676. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3677. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3678. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3679. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3680. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3681. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3682. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3683. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3684. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3685. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3686. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3687. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3688. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3689. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3690.  
  3691. 63.3% (316.3ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3394][search][T#5]'
  3692. 5/10 snapshots sharing following 24 elements
  3693. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3694. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3695. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3696. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3697. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3698. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3699. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3700. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3701. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3702. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3703. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3704. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3705. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3706. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3707. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3708. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3709. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3710. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3711. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3712. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3713. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3714. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3715. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3716. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3717. 5/10 snapshots sharing following 22 elements
  3718. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3719. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3720. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3721. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3722. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3723. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3724. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3725. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3726. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3727. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3728. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3729. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3730. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3731. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3732. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3733. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3734. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3735. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3736. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3737. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3738. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3739. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3740.  
  3741. 62.8% (314ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3394][search][T#38]'
  3742. 2/10 snapshots sharing following 32 elements
  3743. app//org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:40)
  3744. app//org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:194)
  3745. app//org.apache.lucene.search.FilterScorer.score(FilterScorer.java:59)
  3746. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:388)
  3747. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3748. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3749. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3750. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3751. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3752. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3753. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3754. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3755. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3756. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3757. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3758. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3759. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3760. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3761. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3762. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3763. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3764. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3765. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3766. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3767. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3768. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3769. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3770. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3771. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3772. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3773. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3774. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3775. 2/10 snapshots sharing following 25 elements
  3776. app//org.apache.lucene.search.ConjunctionDISI.doNext(ConjunctionDISI.java:200)
  3777. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3778. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3779. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3780. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3781. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3782. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3783. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3784. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3785. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3786. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3787. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3788. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3789. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3790. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3791. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3792. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3793. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3794. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3795. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3796. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3797. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3798. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3799. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3800. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3801. 3/10 snapshots sharing following 31 elements
  3802. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3803. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3804. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3805. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3806. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3807. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3808. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3809. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3810. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3811. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3812. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3813. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3814. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3815. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3816. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3817. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3818. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3819. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3820. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3821. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3822. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3823. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3824. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3825. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3826. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3827. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3828. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3829. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3830. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3831. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3832. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3833. 2/10 snapshots sharing following 24 elements
  3834. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3835. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3836. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3837. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3838. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3839. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3840. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3841. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3842. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3843. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3844. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3845. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3846. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3847. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3848. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3849. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3850. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3851. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3852. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3853. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3854. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3855. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3856. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3857. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3858. unique snapshot
  3859. java.base@13.0.2/java.nio.MappedByteBuffer.limit(MappedByteBuffer.java:332)
  3860. java.base@13.0.2/java.nio.MappedByteBuffer.limit(MappedByteBuffer.java:68)
  3861. java.base@13.0.2/java.nio.Buffer.<init>(Buffer.java:223)
  3862. java.base@13.0.2/java.nio.ByteBuffer.<init>(ByteBuffer.java:284)
  3863. java.base@13.0.2/java.nio.ByteBuffer.<init>(ByteBuffer.java:292)
  3864. java.base@13.0.2/java.nio.MappedByteBuffer.<init>(MappedByteBuffer.java:91)
  3865. java.base@13.0.2/java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:193)
  3866. java.base@13.0.2/java.nio.DirectByteBufferR.<init>(DirectByteBufferR.java:201)
  3867. java.base@13.0.2/java.nio.DirectByteBufferR.slice(DirectByteBufferR.java:218)
  3868. app//org.apache.lucene.store.ByteBufferIndexInput.newCloneInstance(ByteBufferIndexInput.java:339)
  3869. app//org.apache.lucene.store.ByteBufferIndexInput.buildSlice(ByteBufferIndexInput.java:328)
  3870. app//org.apache.lucene.store.ByteBufferIndexInput.slice(ByteBufferIndexInput.java:316)
  3871. app//org.apache.lucene.store.ByteBufferIndexInput$SingleBufferImpl.slice(ByteBufferIndexInput.java:396)
  3872. app//org.apache.lucene.store.IndexInput.randomAccessSlice(IndexInput.java:122)
  3873. app//org.apache.lucene.util.fst.OffHeapFSTStore.getReverseBytesReader(OffHeapFSTStore.java:64)
  3874. app//org.apache.lucene.util.fst.FST.getBytesReader(FST.java:1442)
  3875. app//org.apache.lucene.codecs.blocktree.SegmentTermsEnum.<init>(SegmentTermsEnum.java:82)
  3876. app//org.apache.lucene.codecs.blocktree.FieldReader.iterator(FieldReader.java:176)
  3877. app//org.apache.lucene.search.TermQuery$TermWeight.getTermsEnum(TermQuery.java:139)
  3878. app//org.apache.lucene.search.TermQuery$TermWeight.scorer(TermQuery.java:109)
  3879. app//org.apache.lucene.search.Weight.scorerSupplier(Weight.java:147)
  3880. app//org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:379)
  3881. app//org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:344)
  3882. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$CustomBoostFactorWeight.functionScorer(FunctionScoreQuery.java:268)
  3883. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$CustomBoostFactorWeight.scorer(FunctionScoreQuery.java:291)
  3884. app//org.apache.lucene.search.Weight.scorerSupplier(Weight.java:147)
  3885. app//org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:379)
  3886. app//org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:344)
  3887. app//org.apache.lucene.search.Weight.bulkScorer(Weight.java:181)
  3888. app//org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:338)
  3889. app//org.elasticsearch.search.internal.ContextIndexSearcher$1.bulkScorer(ContextIndexSearcher.java:244)
  3890. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:195)
  3891. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3892. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3893. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3894. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3895. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3896. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3897. app//org.elasticsearch.search.SearchService$$Lambda$4480/0x0000000801894440.get(Unknown Source)
  3898. app//org.elasticsearch.search.SearchService$$Lambda$4483/0x0000000801897040.get(Unknown Source)
  3899. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3900. app//org.elasticsearch.action.ActionRunnable$$Lambda$4429/0x000000080187f040.accept(Unknown Source)
  3901. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3902. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3903. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3904. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3905. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3906. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3907. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3908. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3909.  
  3910. ::: {chi1-lapp2677}{y7CHBr1CRkqz0bvNO3L_YQ}{zGBxbmK-TbWqiuY96TVV7A}{10.128.170.12}{10.128.170.12:9300}{dil}{ml.machine_memory=126564622336, ml.max_open_jobs=20, xpack.installed=true}
  3911. Hot threads at 2020-09-05T21:56:41.029Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3912.  
  3913. ::: {chi1-lapp3222}{87yol3x1Q_esfJjo2ZVM_g}{Ht9jLLnTSUqylcMeBefqVg}{10.128.167.130}{10.128.167.130:9300}{dil}{ml.machine_memory=126564569088, ml.max_open_jobs=20, datacenter=rbx-chi1, xpack.installed=true}
  3914. Hot threads at 2020-09-05T21:56:41.031Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3915.  
  3916. 29.1% (145.4ms out of 500ms) cpu usage by thread 'elasticsearch[chi1-lapp3222][search][T#14]'
  3917. 3/10 snapshots sharing following 31 elements
  3918. app//org.elasticsearch.common.lucene.search.function.ScriptScoreFunction$1.score(ScriptScoreFunction.java:81)
  3919. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.computeScore(FunctionScoreQuery.java:410)
  3920. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:392)
  3921. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3922. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3923. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3924. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3925. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3926. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3927. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3928. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3929. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3930. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3931. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3932. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3933. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3934. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3935. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3936. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3937. app//org.elasticsearch.search.SearchService$$Lambda$4528/0x000000080189f840.get(Unknown Source)
  3938. app//org.elasticsearch.search.SearchService$$Lambda$4534/0x000000080189ec40.get(Unknown Source)
  3939. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3940. app//org.elasticsearch.action.ActionRunnable$$Lambda$4453/0x0000000801882c40.accept(Unknown Source)
  3941. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3942. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3943. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3944. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3945. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3946. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3947. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3948. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3949. 2/10 snapshots sharing following 24 elements
  3950. app//org.apache.lucene.search.ConjunctionDISI.nextDoc(ConjunctionDISI.java:240)
  3951. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:244)
  3952. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3953. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3954. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3955. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3956. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3957. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3958. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3959. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3960. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3961. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3962. app//org.elasticsearch.search.SearchService$$Lambda$4528/0x000000080189f840.get(Unknown Source)
  3963. app//org.elasticsearch.search.SearchService$$Lambda$4534/0x000000080189ec40.get(Unknown Source)
  3964. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  3965. app//org.elasticsearch.action.ActionRunnable$$Lambda$4453/0x0000000801882c40.accept(Unknown Source)
  3966. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  3967. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3968. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  3969. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  3970. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  3971. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  3972. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  3973. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  3974. 2/10 snapshots sharing following 35 elements
  3975. app//org.apache.lucene.search.TermScorer.score(TermScorer.java:75)
  3976. app//org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:40)
  3977. app//org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:194)
  3978. app//org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:40)
  3979. app//org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:194)
  3980. app//org.apache.lucene.search.FilterScorer.score(FilterScorer.java:59)
  3981. app//org.elasticsearch.common.lucene.search.function.FunctionScoreQuery$FunctionFactorScorer.score(FunctionScoreQuery.java:388)
  3982. app//org.apache.lucene.search.ConjunctionScorer.score(ConjunctionScorer.java:59)
  3983. app//org.apache.lucene.search.ScoreCachingWrappingScorer.score(ScoreCachingWrappingScorer.java:50)
  3984. app//org.apache.lucene.search.FieldComparator$RelevanceComparator.compareBottom(FieldComparator.java:430)
  3985. app//org.apache.lucene.search.MultiLeafFieldComparator.compareBottom(MultiLeafFieldComparator.java:55)
  3986. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  3987. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  3988. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  3989. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  3990. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  3991. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  3992. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  3993. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  3994. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  3995. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  3996. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  3997. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  3998. app//org.elasticsearch.search.SearchService$$Lambda$4528/0x000000080189f840.get(Unknown Source)
  3999. app//org.elasticsearch.search.SearchService$$Lambda$4534/0x000000080189ec40.get(Unknown Source)
  4000. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  4001. app//org.elasticsearch.action.ActionRunnable$$Lambda$4453/0x0000000801882c40.accept(Unknown Source)
  4002. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  4003. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  4004. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  4005. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  4006. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  4007. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  4008. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  4009. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  4010. 3/10 snapshots sharing following 24 elements
  4011. app//org.apache.lucene.search.TopFieldCollector$SimpleFieldCollector$1.collect(TopFieldCollector.java:145)
  4012. app//org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:242)
  4013. app//org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:229)
  4014. app//org.elasticsearch.search.internal.CancellableBulkScorer.score(CancellableBulkScorer.java:56)
  4015. app//org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  4016. app//org.elasticsearch.search.internal.ContextIndexSearcher.searchLeaf(ContextIndexSearcher.java:198)
  4017. app//org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:171)
  4018. app//org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:445)
  4019. app//org.elasticsearch.search.query.QueryPhase.searchWithCollector(QueryPhase.java:333)
  4020. app//org.elasticsearch.search.query.QueryPhase.executeInternal(QueryPhase.java:295)
  4021. app//org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:134)
  4022. app//org.elasticsearch.search.SearchService.lambda$executeQueryPhase$3(SearchService.java:430)
  4023. app//org.elasticsearch.search.SearchService$$Lambda$4528/0x000000080189f840.get(Unknown Source)
  4024. app//org.elasticsearch.search.SearchService$$Lambda$4534/0x000000080189ec40.get(Unknown Source)
  4025. app//org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:58)
  4026. app//org.elasticsearch.action.ActionRunnable$$Lambda$4453/0x0000000801882c40.accept(Unknown Source)
  4027. app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73)
  4028. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  4029. app//org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:44)
  4030. app//org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:692)
  4031. app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  4032. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
  4033. java.base@13.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
  4034. java.base@13.0.2/java.lang.Thread.run(Thread.java:830)
  4035.  
  4036.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!