API tools faq
paste
Guest User

Output hot threads

a guest
Sep 21st, 2017
115
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 98.93 KB | None | 0 0
raw download clone embed print report
  1. ::: {cls-es-slave4}{_0OHJK79RmW0JjVsacr--w}{10.240.0.56}{10.240.0.56:9300}{master=false}
  2. Hot threads at 2017-09-21T06:28:25.614Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3.  
  4. 100.5% (502.5ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave4][clusterService#updateTask][T#1]'
  5. 6/10 snapshots sharing following 73 elements
  6. org.apache.lucene.util.fst.FST.readFirstRealTargetArc(FST.java:992)
  7. org.apache.lucene.util.fst.NodeHash.hash(NodeHash.java:98)
  8. org.apache.lucene.util.fst.NodeHash.addNew(NodeHash.java:148)
  9. org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
  10. org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
  11. org.apache.lucene.util.fst.Builder.compileNode(Builder.java:215)
  12. org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
  13. org.apache.lucene.util.fst.Builder.add(Builder.java:417)
  14. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  15. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  16. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  17. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  18. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  19. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  20. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  21. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  22. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  23. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  24. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  25. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  26. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  27. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  28. com.sun.proxy.$Proxy15.create(Unknown Source)
  29. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  30. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  31. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  32. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  33. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  34. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  35. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  36. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  37. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  38. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  39. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  40. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  41. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  42. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  43. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  44. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  45. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  46. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  47. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  48. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  49. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  50. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  51. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  52. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  53. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  54. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  55. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  56. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  57. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  58. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  59. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  60. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  61. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  62. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  63. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  64. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  65. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  66. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  67. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  68. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  69. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  70. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  71. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  72. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  73. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  74. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  75. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  76. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  77. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  78. java.lang.Thread.run(Thread.java:748)
  79. 2/10 snapshots sharing following 72 elements
  80. org.apache.lucene.util.packed.GrowableWriter.get(GrowableWriter.java:56)
  81. org.apache.lucene.util.packed.AbstractPagedMutable.get(AbstractPagedMutable.java:93)
  82. org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:167)
  83. org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
  84. org.apache.lucene.util.fst.Builder.compileNode(Builder.java:215)
  85. org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
  86. org.apache.lucene.util.fst.Builder.add(Builder.java:417)
  87. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  88. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  89. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  90. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  91. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  92. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  93. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  94. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  95. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  96. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  97. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  98. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  99. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  100. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  101. com.sun.proxy.$Proxy15.create(Unknown Source)
  102. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  103. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  104. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  105. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  106. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  107. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  108. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  109. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  110. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  111. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  112. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  113. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  114. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  115. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  116. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  117. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  118. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  119. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  120. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  121. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  122. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  123. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  124. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  125. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  126. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  127. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  128. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  129. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  130. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  131. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  132. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  133. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  134. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  135. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  136. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  137. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  138. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  139. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  140. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  141. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  142. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  143. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  144. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  145. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  146. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  147. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  148. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  149. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  150. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  151. java.lang.Thread.run(Thread.java:748)
  152. 2/10 snapshots sharing following 70 elements
  153. org.apache.lucene.util.fst.NodeHash.rehash(NodeHash.java:169)
  154. org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:133)
  155. org.apache.lucene.util.fst.Builder.compileNode(Builder.java:215)
  156. org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
  157. org.apache.lucene.util.fst.Builder.add(Builder.java:417)
  158. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  159. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  160. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  161. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  162. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  163. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  164. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  165. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  166. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  167. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  168. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  169. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  170. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  171. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  172. com.sun.proxy.$Proxy15.create(Unknown Source)
  173. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  174. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  175. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  176. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  177. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  178. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  179. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  180. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  181. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  182. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  183. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  184. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  185. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  186. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  187. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  188. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  189. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  190. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  191. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  192. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  193. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  194. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  195. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  196. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  197. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  198. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  199. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  200. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  201. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  202. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  203. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  204. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  205. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  206. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  207. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  208. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  209. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  210. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  211. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  212. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  213. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  214. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  215. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  216. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  217. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  218. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  219. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  220. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  221. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  222. java.lang.Thread.run(Thread.java:748)
  223.  
  224. 0.1% (328micros out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave4][transport_client_timer][T#1]{Hashed wheel timer #1}'
  225. 10/10 snapshots sharing following 5 elements
  226. java.lang.Thread.sleep(Native Method)
  227. org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445)
  228. org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364)
  229. org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
  230. java.lang.Thread.run(Thread.java:748)
  231.  
  232. ::: {cls-es-slave3}{5Xtpez6lQYKWbSIda4R2mA}{10.240.0.54}{10.240.0.54:9300}{master=false}
  233. Hot threads at 2017-09-21T06:29:13.935Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  234.  
  235. 96.8% (484ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave3][search][T#24]'
  236. 3/10 snapshots sharing following 24 elements
  237. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$2.get(Lucene54DocValuesProducer.java:502)
  238. org.apache.lucene.util.LongValues.get(LongValues.java:45)
  239. org.apache.lucene.index.SingletonSortedNumericDocValues.setDocument(SingletonSortedNumericDocValues.java:52)
  240. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator$1.collect(LongTermsAggregator.java:85)
  241. org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:88)
  242. org.apache.lucene.search.MultiCollector$MultiLeafCollector.collect(MultiCollector.java:174)
  243. org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:221)
  244. org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:172)
  245. org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  246. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:821)
  247. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  248. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  249. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  250. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  251. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  252. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  253. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  254. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  255. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  256. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  257. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  258. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  259. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  260. java.lang.Thread.run(Thread.java:748)
  261. 3/10 snapshots sharing following 14 elements
  262. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  263. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  264. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  265. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  266. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  267. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  268. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  269. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  270. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  271. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  272. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  273. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  274. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  275. java.lang.Thread.run(Thread.java:748)
  276. 3/10 snapshots sharing following 26 elements
  277. java.util.LinkedList.listIterator(LinkedList.java:868)
  278. java.util.AbstractList.listIterator(AbstractList.java:299)
  279. java.util.AbstractSequentialList.iterator(AbstractSequentialList.java:239)
  280. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:279)
  281. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:266)
  282. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:37)
  283. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:26)
  284. org.apache.lucene.util.PriorityQueue.upHeap(PriorityQueue.java:263)
  285. org.apache.lucene.util.PriorityQueue.add(PriorityQueue.java:140)
  286. org.apache.lucene.util.PriorityQueue.insertWithOverflow(PriorityQueue.java:156)
  287. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:141)
  288. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  289. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  290. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  291. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  292. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  293. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  294. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  295. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  296. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  297. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  298. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  299. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  300. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  301. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  302. java.lang.Thread.run(Thread.java:748)
  303. unique snapshot
  304. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:134)
  305. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  306. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  307. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  308. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  309. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  310. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  311. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  312. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  313. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  314. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  315. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  316. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  317. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  318. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  319. java.lang.Thread.run(Thread.java:748)
  320.  
  321. 96.2% (480.8ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave3][search][T#16]'
  322. 2/10 snapshots sharing following 34 elements
  323. sun.nio.ch.FileDispatcherImpl.pread0(Native Method)
  324. sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:52)
  325. sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:220)
  326. sun.nio.ch.IOUtil.read(IOUtil.java:197)
  327. sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:741)
  328. sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:727)
  329. org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179)
  330. org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342)
  331. org.apache.lucene.store.BufferedIndexInput.readLong(BufferedIndexInput.java:315)
  332. org.apache.lucene.util.packed.DirectReader$DirectPackedReader64.get(DirectReader.java:343)
  333. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$2.get(Lucene54DocValuesProducer.java:502)
  334. org.apache.lucene.util.LongValues.get(LongValues.java:45)
  335. org.apache.lucene.index.SingletonSortedNumericDocValues.setDocument(SingletonSortedNumericDocValues.java:52)
  336. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator$1.collect(LongTermsAggregator.java:85)
  337. org.elasticsearch.search.aggregations.LeafBucketCollector.collect(LeafBucketCollector.java:88)
  338. org.apache.lucene.search.MultiCollector$MultiLeafCollector.collect(MultiCollector.java:174)
  339. org.apache.lucene.search.Weight$DefaultBulkScorer.scoreAll(Weight.java:221)
  340. org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:172)
  341. org.apache.lucene.search.BulkScorer.score(BulkScorer.java:39)
  342. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:821)
  343. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  344. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  345. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  346. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  347. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  348. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  349. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  350. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  351. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  352. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  353. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  354. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  355. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  356. java.lang.Thread.run(Thread.java:748)
  357. 3/10 snapshots sharing following 16 elements
  358. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:134)
  359. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  360. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  361. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  362. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  363. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  364. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  365. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  366. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  367. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  368. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  369. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  370. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  371. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  372. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  373. java.lang.Thread.run(Thread.java:748)
  374. 5/10 snapshots sharing following 26 elements
  375. java.util.LinkedList.listIterator(LinkedList.java:868)
  376. java.util.AbstractList.listIterator(AbstractList.java:299)
  377. java.util.AbstractSequentialList.iterator(AbstractSequentialList.java:239)
  378. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:279)
  379. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:266)
  380. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:37)
  381. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:26)
  382. org.apache.lucene.util.PriorityQueue.upHeap(PriorityQueue.java:263)
  383. org.apache.lucene.util.PriorityQueue.add(PriorityQueue.java:140)
  384. org.apache.lucene.util.PriorityQueue.insertWithOverflow(PriorityQueue.java:156)
  385. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:141)
  386. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  387. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  388. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  389. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  390. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  391. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  392. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  393. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  394. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  395. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  396. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  397. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  398. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  399. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  400. java.lang.Thread.run(Thread.java:748)
  401.  
  402. 94.6% (472.7ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave3][search][T#11]'
  403. 9/10 snapshots sharing following 26 elements
  404. java.util.LinkedList.listIterator(LinkedList.java:868)
  405. java.util.AbstractList.listIterator(AbstractList.java:299)
  406. java.util.AbstractSequentialList.iterator(AbstractSequentialList.java:239)
  407. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:279)
  408. org.elasticsearch.search.aggregations.bucket.terms.InternalOrder$CompoundOrder$CompoundOrderComparator.compare(InternalOrder.java:266)
  409. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:37)
  410. org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue.lessThan(BucketPriorityQueue.java:26)
  411. org.apache.lucene.util.PriorityQueue.upHeap(PriorityQueue.java:263)
  412. org.apache.lucene.util.PriorityQueue.add(PriorityQueue.java:140)
  413. org.apache.lucene.util.PriorityQueue.insertWithOverflow(PriorityQueue.java:156)
  414. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:141)
  415. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  416. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  417. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  418. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  419. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  420. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  421. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  422. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  423. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  424. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  425. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  426. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  427. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  428. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  429. java.lang.Thread.run(Thread.java:748)
  430. unique snapshot
  431. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:134)
  432. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  433. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  434. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  435. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  436. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  437. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  438. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  439. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  440. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  441. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  442. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  443. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  444. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  445. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  446. java.lang.Thread.run(Thread.java:748)
  447.  
  448. ::: {cls-es-slave1}{A5-D0jDSQwGGZy7J0RpUQQ}{10.240.0.18}{10.240.0.18:9300}{master=true}
  449. Hot threads at 2017-09-21T06:28:25.614Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  450.  
  451. 100.5% (502.5ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave1][clusterService#updateTask][T#1]'
  452. 3/10 snapshots sharing following 68 elements
  453. org.apache.lucene.util.fst.Builder.compileNode(Builder.java:215)
  454. org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
  455. org.apache.lucene.util.fst.Builder.add(Builder.java:417)
  456. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  457. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  458. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  459. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  460. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  461. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  462. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  463. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  464. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  465. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  466. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  467. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  468. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  469. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  470. com.sun.proxy.$Proxy15.create(Unknown Source)
  471. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  472. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  473. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  474. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  475. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  476. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  477. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  478. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  479. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  480. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  481. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  482. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  483. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  484. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  485. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  486. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  487. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  488. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  489. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  490. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  491. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  492. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  493. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  494. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  495. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  496. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  497. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  498. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  499. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  500. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  501. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  502. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  503. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  504. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  505. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  506. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  507. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  508. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  509. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  510. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  511. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  512. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  513. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  514. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  515. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  516. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  517. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  518. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  519. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  520. java.lang.Thread.run(Thread.java:748)
  521. 2/10 snapshots sharing following 68 elements
  522. java.util.HashMap.putVal(HashMap.java:630)
  523. java.util.HashMap.put(HashMap.java:611)
  524. java.util.HashSet.add(HashSet.java:219)
  525. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:258)
  526. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  527. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  528. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  529. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  530. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  531. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  532. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  533. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  534. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  535. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  536. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  537. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  538. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  539. com.sun.proxy.$Proxy15.create(Unknown Source)
  540. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  541. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  542. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  543. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  544. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  545. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  546. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  547. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  548. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  549. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  550. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  551. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  552. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  553. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  554. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  555. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  556. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  557. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  558. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  559. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  560. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  561. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  562. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  563. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  564. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  565. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  566. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  567. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  568. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  569. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  570. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  571. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  572. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  573. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  574. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  575. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  576. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  577. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  578. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  579. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  580. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  581. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  582. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  583. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  584. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  585. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  586. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  587. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  588. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  589. java.lang.Thread.run(Thread.java:748)
  590. 2/10 snapshots sharing following 67 elements
  591. org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:310)
  592. org.apache.lucene.util.fst.Builder.add(Builder.java:417)
  593. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  594. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  595. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  596. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  597. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  598. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  599. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  600. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  601. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  602. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  603. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  604. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  605. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  606. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  607. com.sun.proxy.$Proxy15.create(Unknown Source)
  608. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  609. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  610. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  611. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  612. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  613. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  614. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  615. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  616. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  617. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  618. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  619. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  620. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  621. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  622. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  623. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  624. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  625. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  626. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  627. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  628. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  629. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  630. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  631. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  632. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  633. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  634. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  635. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  636. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  637. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  638. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  639. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  640. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  641. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  642. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  643. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  644. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  645. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  646. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  647. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  648. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  649. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  650. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  651. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  652. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  653. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  654. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  655. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  656. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  657. java.lang.Thread.run(Thread.java:748)
  658. 2/10 snapshots sharing following 66 elements
  659. org.apache.lucene.util.fst.Builder.add(Builder.java:420)
  660. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  661. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  662. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  663. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  664. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  665. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  666. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  667. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  668. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  669. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  670. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  671. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  672. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  673. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  674. com.sun.proxy.$Proxy15.create(Unknown Source)
  675. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  676. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  677. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  678. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  679. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  680. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  681. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  682. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  683. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  684. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  685. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  686. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  687. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  688. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  689. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  690. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  691. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  692. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  693. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  694. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  695. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  696. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  697. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  698. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  699. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  700. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  701. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  702. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  703. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  704. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  705. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  706. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  707. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  708. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  709. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  710. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  711. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  712. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  713. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  714. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  715. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  716. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  717. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  718. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  719. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  720. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  721. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  722. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  723. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  724. java.lang.Thread.run(Thread.java:748)
  725. unique snapshot
  726. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:280)
  727. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  728. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  729. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  730. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  731. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  732. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  733. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  734. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  735. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  736. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  737. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  738. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  739. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  740. com.sun.proxy.$Proxy15.create(Unknown Source)
  741. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  742. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  743. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  744. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  745. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  746. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  747. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  748. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  749. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  750. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  751. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  752. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  753. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  754. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  755. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  756. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  757. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  758. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  759. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  760. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  761. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  762. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  763. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  764. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  765. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  766. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  767. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  768. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  769. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  770. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  771. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  772. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  773. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  774. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  775. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  776. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  777. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  778. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  779. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  780. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  781. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  782. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  783. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  784. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  785. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  786. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  787. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  788. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  789. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  790. java.lang.Thread.run(Thread.java:748)
  791.  
  792. 0.0% (243micros out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave1][transport_client_timer][T#1]{Hashed wheel timer #1}'
  793. 10/10 snapshots sharing following 5 elements
  794. java.lang.Thread.sleep(Native Method)
  795. org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445)
  796. org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364)
  797. org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
  798. java.lang.Thread.run(Thread.java:748)
  799.  
  800. ::: {cls-es-slave2}{rGOOhoZHSzGp7yNKFPFO9Q}{10.240.0.23}{10.240.0.23:9300}{master=false}
  801. Hot threads at 2017-09-21T06:29:11.366Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  802.  
  803. 84.1% (420.6ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave2][search][T#25]'
  804. 5/10 snapshots sharing following 17 elements
  805. org.elasticsearch.search.aggregations.bucket.BucketsAggregator.bucketAggregations(BucketsAggregator.java:118)
  806. org.elasticsearch.search.aggregations.bucket.terms.LongTermsAggregator.buildAggregation(LongTermsAggregator.java:159)
  807. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:59)
  808. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  809. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  810. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  811. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  812. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  813. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  814. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  815. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  816. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  817. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  818. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  819. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  820. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  821. java.lang.Thread.run(Thread.java:748)
  822. 4/10 snapshots sharing following 14 elements
  823. org.elasticsearch.search.aggregations.bucket.terms.DoubleTermsAggregator.buildAggregation(DoubleTermsAggregator.java:42)
  824. org.elasticsearch.search.aggregations.AggregationPhase.execute(AggregationPhase.java:167)
  825. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:119)
  826. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  827. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  828. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  829. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  830. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  831. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  832. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  833. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  834. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  835. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  836. java.lang.Thread.run(Thread.java:748)
  837. unique snapshot
  838. java.util.AbstractList.iterator(AbstractList.java:288)
  839. org.elasticsearch.search.aggregations.InternalAggregations.writeTo(InternalAggregations.java:229)
  840. org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms$Bucket.writeTo(DoubleTerms.java:136)
  841. org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms.doWriteTo(DoubleTerms.java:233)
  842. org.elasticsearch.search.aggregations.InternalAggregation.writeTo(InternalAggregation.java:220)
  843. org.elasticsearch.search.aggregations.InternalAggregations.writeTo(InternalAggregations.java:232)
  844. org.elasticsearch.search.query.QuerySearchResult.writeToNoId(QuerySearchResult.java:241)
  845. org.elasticsearch.search.query.QuerySearchResult.writeTo(QuerySearchResult.java:229)
  846. org.elasticsearch.transport.netty.NettyTransportChannel.sendResponse(NettyTransportChannel.java:104)
  847. org.elasticsearch.transport.netty.NettyTransportChannel.sendResponse(NettyTransportChannel.java:80)
  848. org.elasticsearch.transport.DelegatingTransportChannel.sendResponse(DelegatingTransportChannel.java:58)
  849. org.elasticsearch.transport.RequestHandlerRegistry$TransportChannelWrapper.sendResponse(RequestHandlerRegistry.java:140)
  850. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:369)
  851. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  852. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  853. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  854. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  855. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  856. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  857. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  858. java.lang.Thread.run(Thread.java:748)
  859.  
  860. 28.5% (142.4ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-slave2][search][T#14]'
  861. 2/10 snapshots sharing following 26 elements
  862. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1285)
  863. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1186)
  864. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.pushFrame(IntersectTermsEnum.java:202)
  865. org.apache.lucene.codecs.blocktree.IntersectTermsEnum._next(IntersectTermsEnum.java:666)
  866. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:501)
  867. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:181)
  868. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:213)
  869. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  870. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  871. org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
  872. org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:370)
  873. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:818)
  874. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  875. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  876. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  877. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  878. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  879. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  880. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  881. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  882. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  883. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  884. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  885. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  886. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  887. java.lang.Thread.run(Thread.java:748)
  888. 3/10 snapshots sharing following 30 elements
  889. org.apache.lucene.util.fst.ByteSequenceOutputs.read(ByteSequenceOutputs.java:129)
  890. org.apache.lucene.util.fst.ByteSequenceOutputs.read(ByteSequenceOutputs.java:35)
  891. org.apache.lucene.util.fst.Outputs.readFinalOutput(Outputs.java:77)
  892. org.apache.lucene.util.fst.FST.readNextRealArc(FST.java:1094)
  893. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1262)
  894. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1186)
  895. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.pushFrame(IntersectTermsEnum.java:202)
  896. org.apache.lucene.codecs.blocktree.IntersectTermsEnum._next(IntersectTermsEnum.java:666)
  897. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:501)
  898. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:181)
  899. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:213)
  900. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  901. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  902. org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
  903. org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:370)
  904. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:818)
  905. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  906. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  907. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  908. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  909. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  910. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  911. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  912. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  913. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  914. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  915. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  916. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  917. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  918. java.lang.Thread.run(Thread.java:748)
  919. 2/10 snapshots sharing following 30 elements
  920. org.apache.lucene.util.fst.ByteSequenceOutputs.read(ByteSequenceOutputs.java:129)
  921. org.apache.lucene.util.fst.ByteSequenceOutputs.read(ByteSequenceOutputs.java:35)
  922. org.apache.lucene.util.fst.FST.readNextRealArc(FST.java:1088)
  923. org.apache.lucene.util.fst.FST.readFirstRealTargetArc(FST.java:992)
  924. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1270)
  925. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1186)
  926. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.pushFrame(IntersectTermsEnum.java:202)
  927. org.apache.lucene.codecs.blocktree.IntersectTermsEnum._next(IntersectTermsEnum.java:666)
  928. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:501)
  929. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:181)
  930. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:213)
  931. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  932. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  933. org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
  934. org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:370)
  935. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:818)
  936. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  937. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  938. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  939. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  940. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  941. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  942. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  943. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  944. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  945. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  946. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  947. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  948. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  949. java.lang.Thread.run(Thread.java:748)
  950. 3/10 snapshots sharing following 25 elements
  951. org.apache.lucene.util.fst.FST.findTargetArc(FST.java:1186)
  952. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.pushFrame(IntersectTermsEnum.java:202)
  953. org.apache.lucene.codecs.blocktree.IntersectTermsEnum._next(IntersectTermsEnum.java:666)
  954. org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:501)
  955. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:181)
  956. org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:213)
  957. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  958. org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:389)
  959. org.apache.lucene.search.Weight.bulkScorer(Weight.java:135)
  960. org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:370)
  961. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:818)
  962. org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:535)
  963. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:384)
  964. org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:113)
  965. org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:372)
  966. org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:385)
  967. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:368)
  968. org.elasticsearch.search.action.SearchServiceTransportAction$SearchQueryTransportHandler.messageReceived(SearchServiceTransportAction.java:365)
  969. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:33)
  970. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:77)
  971. org.elasticsearch.transport.netty.MessageChannelHandler$RequestHandler.doRun(MessageChannelHandler.java:293)
  972. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  973. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  974. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  975. java.lang.Thread.run(Thread.java:748)
  976.  
  977. ::: {cls-es-master}{OC4t1qzpQM6Jyi9TvReS-Q}{10.240.0.15}{10.240.0.15:9300}{master=true}
  978. Hot threads at 2017-09-21T06:28:25.614Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  979.  
  980. 100.4% (502ms out of 500ms) cpu usage by thread 'elasticsearch[cls-es-master][clusterService#updateTask][T#1]'
  981. 10/10 snapshots sharing following 70 elements
  982. java.util.TimSort.mergeHi(TimSort.java:850)
  983. java.util.TimSort.mergeAt(TimSort.java:516)
  984. java.util.TimSort.mergeForceCollapse(TimSort.java:457)
  985. java.util.TimSort.sort(TimSort.java:254)
  986. java.util.Arrays.sort(Arrays.java:1438)
  987. org.apache.lucene.analysis.synonym.SynonymMap$Builder.build(SynonymMap.java:233)
  988. org.elasticsearch.index.analysis.SynonymTokenFilterFactory.<init>(SynonymTokenFilterFactory.java:104)
  989. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  990. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  991. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  992. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  993. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  994. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  995. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  996. org.elasticsearch.common.inject.FactoryProxy.get(FactoryProxy.java:54)
  997. org.elasticsearch.common.inject.InjectorImpl$4$1.call(InjectorImpl.java:823)
  998. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  999. org.elasticsearch.common.inject.InjectorImpl$4.get(InjectorImpl.java:818)
  1000. org.elasticsearch.common.inject.assistedinject.FactoryProvider2.invoke(FactoryProvider2.java:236)
  1001. com.sun.proxy.$Proxy15.create(Unknown Source)
  1002. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:151)
  1003. org.elasticsearch.index.analysis.AnalysisService.<init>(AnalysisService.java:70)
  1004. sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  1005. sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  1006. sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  1007. java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  1008. org.elasticsearch.common.inject.DefaultConstructionProxyFactory$1.newInstance(DefaultConstructionProxyFactory.java:50)
  1009. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:86)
  1010. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  1011. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  1012. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  1013. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  1014. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  1015. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  1016. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  1017. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  1018. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  1019. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  1020. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  1021. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  1022. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  1023. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  1024. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  1025. org.elasticsearch.common.inject.SingleParameterInjector.inject(SingleParameterInjector.java:42)
  1026. org.elasticsearch.common.inject.SingleParameterInjector.getAll(SingleParameterInjector.java:66)
  1027. org.elasticsearch.common.inject.ConstructorInjector.construct(ConstructorInjector.java:85)
  1028. org.elasticsearch.common.inject.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:104)
  1029. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:47)
  1030. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:886)
  1031. org.elasticsearch.common.inject.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:43)
  1032. org.elasticsearch.common.inject.Scopes$1$1.get(Scopes.java:59)
  1033. org.elasticsearch.common.inject.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:46)
  1034. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:201)
  1035. org.elasticsearch.common.inject.InjectorBuilder$1.call(InjectorBuilder.java:193)
  1036. org.elasticsearch.common.inject.InjectorImpl.callInContext(InjectorImpl.java:879)
  1037. org.elasticsearch.common.inject.InjectorBuilder.loadEagerSingletons(InjectorBuilder.java:193)
  1038. org.elasticsearch.common.inject.InjectorBuilder.injectDynamically(InjectorBuilder.java:175)
  1039. org.elasticsearch.common.inject.InjectorBuilder.build(InjectorBuilder.java:110)
  1040. org.elasticsearch.common.inject.InjectorImpl.createChildInjector(InjectorImpl.java:154)
  1041. org.elasticsearch.common.inject.ModulesBuilder.createChildInjector(ModulesBuilder.java:55)
  1042. org.elasticsearch.indices.IndicesService.createIndex(IndicesService.java:358)
  1043. org.elasticsearch.indices.cluster.IndicesClusterStateService.applyNewIndices(IndicesClusterStateService.java:317)
  1044. org.elasticsearch.indices.cluster.IndicesClusterStateService.clusterChanged(IndicesClusterStateService.java:174)
  1045. org.elasticsearch.cluster.service.InternalClusterService.runTasksForExecutor(InternalClusterService.java:622)
  1046. org.elasticsearch.cluster.service.InternalClusterService$UpdateTask.run(InternalClusterService.java:784)
  1047. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:231)
  1048. org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:194)
  1049. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1050. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1051. java.lang.Thread.run(Thread.java:748)
  1052.  
  1053. 0.1% (255.2micros out of 500ms) cpu usage by thread 'elasticsearch[cls-es-master][transport_client_timer][T#1]{Hashed wheel timer #1}'
  1054. 10/10 snapshots sharing following 5 elements
  1055. java.lang.Thread.sleep(Native Method)
  1056. org.jboss.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:445)
  1057. org.jboss.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:364)
  1058. org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108)
  1059. java.lang.Thread.run(Thread.java:748)
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!