API tools faq
paste
Advertisement
bra_fsn_hu

es 5.1.2 hot_threads

bra_fsn_hu
Jan 31st, 2017
356
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 266.82 KB | None | 0 0
raw download clone embed print report
  1. ::: {esm02}{1T6OkooQQTuX_AiVH85QTQ}{WJWmPAOiTYea8rpJENITng}{10.6.145.102}{10.6.145.102:9300}
  2. Hot threads at 2017-01-31T09:48:30.114Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3.  
  4. ::: {fe05}{1qbkkgl4R4CV3Moi7wFp-A}{Ylk1GfszS6mKWWRWj0OH2g}{10.6.145.198}{10.6.145.198:9300}
  5. Hot threads at 2017-01-31T09:48:30.110Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  6.  
  7. ::: {fe18}{XdeOyod3SZK05e21q07qtQ}{BHsp65SCRFaI5sjl-W6OLA}{10.6.145.211}{10.6.145.211:9300}
  8. Hot threads at 2017-01-31T09:48:30.110Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  9.  
  10. 95.6% (478.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe18][warmer][T#4]'
  11. 10/10 snapshots sharing following 18 elements
  12. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  13. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  14. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  15. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  16. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  17. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  18. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  19. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/2009837051.load(Unknown Source)
  20. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  21. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  22. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  23. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  24. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  25. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/645007284.run(Unknown Source)
  26. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  27. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  28. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  29. java.lang.Thread.run(Thread.java:745)
  30.  
  31. 95.5% (477.6ms out of 500ms) cpu usage by thread 'elasticsearch[fe18][warmer][T#3]'
  32. 8/10 snapshots sharing following 18 elements
  33. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  34. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  35. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  36. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  37. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  38. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  39. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  40. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/2009837051.load(Unknown Source)
  41. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  42. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  43. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  44. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  45. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  46. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/645007284.run(Unknown Source)
  47. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  48. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  49. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  50. java.lang.Thread.run(Thread.java:745)
  51. 2/10 snapshots sharing following 18 elements
  52. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  53. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  54. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  55. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  56. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  57. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  58. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  59. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/2009837051.load(Unknown Source)
  60. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  61. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  62. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  63. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  64. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  65. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/645007284.run(Unknown Source)
  66. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  67. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  68. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  69. java.lang.Thread.run(Thread.java:745)
  70.  
  71. 31.2% (155.9ms out of 500ms) cpu usage by thread 'elasticsearch[fe18][warmer][T#2]'
  72. 6/10 snapshots sharing following 21 elements
  73. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  74. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  75. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  76. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  77. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  78. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  79. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  80. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  81. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  82. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  83. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/2009837051.load(Unknown Source)
  84. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  85. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  86. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  87. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  88. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  89. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/645007284.run(Unknown Source)
  90. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  91. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  92. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  93. java.lang.Thread.run(Thread.java:745)
  94. 4/10 snapshots sharing following 17 elements
  95. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  96. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  97. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  98. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  99. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  100. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  101. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/2009837051.load(Unknown Source)
  102. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  103. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  104. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  105. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  106. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  107. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/645007284.run(Unknown Source)
  108. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  109. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  110. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  111. java.lang.Thread.run(Thread.java:745)
  112.  
  113. ::: {fe09}{HC5iVVBFQf6iQjVn_4uaQQ}{m9rgG2qTR5CICXwnNViEjQ}{10.6.145.202}{10.6.145.202:9300}
  114. Hot threads at 2017-01-31T09:48:30.115Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  115.  
  116. 113.2% (566.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe09][warmer][T#8]'
  117. 3/10 snapshots sharing following 21 elements
  118. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  119. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  120. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  121. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  122. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  123. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  124. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  125. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  126. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  127. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  128. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  129. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  130. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  131. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  132. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  133. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  134. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  135. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  136. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  137. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  138. java.lang.Thread.run(Thread.java:745)
  139. 3/10 snapshots sharing following 21 elements
  140. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  141. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  142. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  143. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  144. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  145. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  146. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  147. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  148. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  149. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  150. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  151. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  152. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  153. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  154. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  155. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  156. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  157. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  158. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  159. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  160. java.lang.Thread.run(Thread.java:745)
  161. 4/10 snapshots sharing following 17 elements
  162. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  163. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  164. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  165. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  166. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  167. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  168. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  169. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  170. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  171. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  172. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  173. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  174. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  175. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  176. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  177. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  178. java.lang.Thread.run(Thread.java:745)
  179.  
  180. 113.2% (566.1ms out of 500ms) cpu usage by thread 'elasticsearch[fe09][warmer][T#6]'
  181. 2/10 snapshots sharing following 21 elements
  182. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1362)
  183. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  184. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  185. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  186. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  187. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  188. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  189. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  190. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  191. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  192. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  193. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  194. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  195. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  196. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  197. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  198. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  199. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  200. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  201. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  202. java.lang.Thread.run(Thread.java:745)
  203. 3/10 snapshots sharing following 21 elements
  204. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  205. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  206. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  207. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  208. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  209. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  210. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  211. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  212. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  213. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  214. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  215. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  216. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  217. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  218. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  219. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  220. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  221. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  222. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  223. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  224. java.lang.Thread.run(Thread.java:745)
  225. 2/10 snapshots sharing following 21 elements
  226. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  227. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  228. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  229. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  230. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  231. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  232. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  233. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  234. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  235. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  236. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  237. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  238. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  239. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  240. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  241. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  242. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  243. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  244. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  245. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  246. java.lang.Thread.run(Thread.java:745)
  247. 3/10 snapshots sharing following 17 elements
  248. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  249. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  250. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  251. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  252. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  253. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  254. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  255. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  256. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  257. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  258. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  259. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  260. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  261. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  262. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  263. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  264. java.lang.Thread.run(Thread.java:745)
  265.  
  266. 113.2% (566ms out of 500ms) cpu usage by thread 'elasticsearch[fe09][warmer][T#4]'
  267. 2/10 snapshots sharing following 21 elements
  268. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  269. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  270. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  271. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  272. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  273. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  274. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  275. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  276. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  277. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  278. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  279. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  280. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  281. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  282. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  283. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  284. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  285. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  286. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  287. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  288. java.lang.Thread.run(Thread.java:745)
  289. 4/10 snapshots sharing following 21 elements
  290. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  291. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  292. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  293. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  294. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  295. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  296. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  297. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  298. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  299. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  300. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  301. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  302. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  303. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  304. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  305. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  306. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  307. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  308. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  309. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  310. java.lang.Thread.run(Thread.java:745)
  311. 3/10 snapshots sharing following 18 elements
  312. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  313. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  314. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  315. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  316. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  317. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  318. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  319. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  320. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  321. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  322. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  323. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  324. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  325. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  326. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  327. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  328. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  329. java.lang.Thread.run(Thread.java:745)
  330. unique snapshot
  331. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  332. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  333. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  334. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  335. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  336. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  337. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  338. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/596078038.load(Unknown Source)
  339. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  340. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  341. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  342. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  343. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  344. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/1266638397.run(Unknown Source)
  345. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  346. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  347. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  348. java.lang.Thread.run(Thread.java:745)
  349.  
  350. ::: {fe12}{9YnNXGnwT9K_jSLmdzW_-g}{S5gElVOaRTm4LnX8UivY2Q}{10.6.145.205}{10.6.145.205:9300}
  351. Hot threads at 2017-01-31T09:48:30.115Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  352.  
  353. 113.2% (566.1ms out of 500ms) cpu usage by thread 'elasticsearch[fe12][warmer][T#4]'
  354. 5/10 snapshots sharing following 21 elements
  355. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  356. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  357. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  358. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  359. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  360. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  361. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  362. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  363. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  364. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  365. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  366. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  367. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  368. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  369. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  370. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  371. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  372. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  373. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  374. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  375. java.lang.Thread.run(Thread.java:745)
  376. 5/10 snapshots sharing following 17 elements
  377. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  378. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  379. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  380. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  381. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  382. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  383. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  384. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  385. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  386. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  387. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  388. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  389. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  390. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  391. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  392. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  393. java.lang.Thread.run(Thread.java:745)
  394.  
  395. 110.8% (553.9ms out of 500ms) cpu usage by thread 'elasticsearch[fe12][warmer][T#8]'
  396. 2/10 snapshots sharing following 22 elements
  397. org.apache.lucene.util.PriorityQueue.downHeap(PriorityQueue.java:284)
  398. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  399. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  400. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  401. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  402. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  403. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  404. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  405. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  406. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  407. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  408. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  409. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  410. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  411. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  412. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  413. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  414. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  415. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  416. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  417. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  418. java.lang.Thread.run(Thread.java:745)
  419. 2/10 snapshots sharing following 21 elements
  420. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  421. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  422. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  423. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  424. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  425. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  426. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  427. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  428. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  429. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  430. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  431. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  432. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  433. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  434. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  435. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  436. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  437. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  438. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  439. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  440. java.lang.Thread.run(Thread.java:745)
  441. 6/10 snapshots sharing following 17 elements
  442. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  443. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  444. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  445. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  446. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  447. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  448. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  449. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  450. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  451. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  452. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  453. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  454. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  455. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  456. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  457. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  458. java.lang.Thread.run(Thread.java:745)
  459.  
  460. 105.9% (529.5ms out of 500ms) cpu usage by thread 'elasticsearch[fe12][warmer][T#9]'
  461. 5/10 snapshots sharing following 19 elements
  462. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  463. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  464. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  465. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  466. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  467. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  468. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  469. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  470. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  471. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  472. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  473. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  474. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  475. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  476. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  477. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  478. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  479. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  480. java.lang.Thread.run(Thread.java:745)
  481. 3/10 snapshots sharing following 21 elements
  482. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  483. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  484. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  485. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  486. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  487. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  488. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  489. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  490. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  491. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  492. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  493. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  494. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  495. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  496. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  497. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  498. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  499. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  500. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  501. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  502. java.lang.Thread.run(Thread.java:745)
  503. 2/10 snapshots sharing following 17 elements
  504. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  505. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  506. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  507. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  508. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  509. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  510. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1371/1586380345.load(Unknown Source)
  511. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  512. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  513. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  514. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  515. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  516. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1368/959411755.run(Unknown Source)
  517. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  518. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  519. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  520. java.lang.Thread.run(Thread.java:745)
  521.  
  522. ::: {fe13}{mTacSSOnRpGV1gyYmlKa6A}{Masy-xhpQRC9zsDRdo3j_w}{10.6.145.206}{10.6.145.206:9300}
  523. Hot threads at 2017-01-31T09:48:30.110Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  524.  
  525. 113.2% (566.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe13][warmer][T#7]'
  526. 10/10 snapshots sharing following 17 elements
  527. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  528. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  529. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  530. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  531. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  532. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  533. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  534. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  535. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  536. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  537. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  538. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  539. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  540. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  541. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  542. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  543. java.lang.Thread.run(Thread.java:745)
  544.  
  545. 111.5% (557.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe13][warmer][T#6]'
  546. 3/10 snapshots sharing following 21 elements
  547. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  548. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  549. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  550. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  551. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  552. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  553. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  554. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  555. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  556. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  557. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  558. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  559. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  560. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  561. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  562. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  563. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  564. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  565. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  566. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  567. java.lang.Thread.run(Thread.java:745)
  568. 3/10 snapshots sharing following 18 elements
  569. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  570. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  571. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  572. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  573. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  574. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  575. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  576. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  577. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  578. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  579. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  580. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  581. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  582. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  583. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  584. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  585. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  586. java.lang.Thread.run(Thread.java:745)
  587. 2/10 snapshots sharing following 21 elements
  588. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  589. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  590. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  591. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  592. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  593. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  594. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  595. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  596. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  597. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  598. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  599. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  600. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  601. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  602. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  603. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  604. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  605. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  606. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  607. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  608. java.lang.Thread.run(Thread.java:745)
  609. 2/10 snapshots sharing following 21 elements
  610. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  611. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  612. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  613. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  614. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  615. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  616. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  617. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  618. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  619. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  620. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  621. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  622. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  623. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  624. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  625. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  626. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  627. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  628. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  629. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  630. java.lang.Thread.run(Thread.java:745)
  631.  
  632. 92.9% (464.6ms out of 500ms) cpu usage by thread 'elasticsearch[fe13][warmer][T#2]'
  633. 7/10 snapshots sharing following 17 elements
  634. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  635. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  636. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  637. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  638. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  639. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  640. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1368/314691759.load(Unknown Source)
  641. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  642. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  643. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  644. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  645. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  646. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/600175617.run(Unknown Source)
  647. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  648. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  649. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  650. java.lang.Thread.run(Thread.java:745)
  651.  
  652. ::: {fe00}{X_QXm-KVRlG_jnHLmieFiA}{7Yt7JAaMSCKYvRk0ZGXB-A}{10.6.145.193}{10.6.145.193:9300}
  653. Hot threads at 2017-01-31T09:48:30.111Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  654.  
  655. ::: {lb01}{HEI15fLWT-avw0mNeeIaqQ}{2axjuNrSQp-4Ge2uoJwMjQ}{10.6.145.223}{10.6.145.223:9300}
  656. Hot threads at 2017-01-31T09:48:30.111Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  657.  
  658. 0.4% (1.9ms out of 500ms) cpu usage by thread 'elasticsearch[lb01][[transport_server_worker.default]][T#70]'
  659. 8/10 snapshots sharing following 2 elements
  660. io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873)
  661. java.lang.Thread.run(Thread.java:745)
  662.  
  663. ::: {fe14}{H3w1mQfPRVOulZXpdM0f1g}{C28jdAEJRNG-kdtBinauzw}{10.6.145.207}{10.6.145.207:9300}
  664. Hot threads at 2017-01-31T09:48:30.111Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  665.  
  666. 75.1% (375.5ms out of 500ms) cpu usage by thread 'elasticsearch[fe14][warmer][T#9]'
  667. 2/10 snapshots sharing following 18 elements
  668. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  669. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  670. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  671. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  672. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  673. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  674. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  675. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  676. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  677. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  678. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  679. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  680. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  681. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  682. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  683. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  684. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  685. java.lang.Thread.run(Thread.java:745)
  686. 4/10 snapshots sharing following 21 elements
  687. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  688. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  689. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  690. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  691. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  692. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  693. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  694. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  695. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  696. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  697. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  698. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  699. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  700. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  701. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  702. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  703. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  704. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  705. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  706. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  707. java.lang.Thread.run(Thread.java:745)
  708. 4/10 snapshots sharing following 21 elements
  709. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  710. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  711. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  712. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  713. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  714. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  715. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  716. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  717. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  718. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  719. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  720. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  721. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  722. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  723. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  724. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  725. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  726. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  727. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  728. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  729. java.lang.Thread.run(Thread.java:745)
  730.  
  731. 75.1% (375.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe14][warmer][T#7]'
  732. 2/10 snapshots sharing following 18 elements
  733. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  734. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  735. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  736. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  737. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  738. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  739. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  740. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  741. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  742. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  743. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  744. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  745. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  746. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  747. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  748. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  749. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  750. java.lang.Thread.run(Thread.java:745)
  751. 3/10 snapshots sharing following 21 elements
  752. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  753. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  754. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  755. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  756. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  757. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  758. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  759. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  760. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  761. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  762. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  763. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  764. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  765. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  766. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  767. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  768. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  769. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  770. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  771. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  772. java.lang.Thread.run(Thread.java:745)
  773. 2/10 snapshots sharing following 21 elements
  774. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  775. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  776. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  777. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  778. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  779. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  780. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  781. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  782. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  783. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  784. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  785. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  786. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  787. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  788. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  789. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  790. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  791. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  792. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  793. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  794. java.lang.Thread.run(Thread.java:745)
  795. 3/10 snapshots sharing following 21 elements
  796. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  797. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  798. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  799. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  800. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  801. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  802. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  803. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  804. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  805. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  806. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1597454377.load(Unknown Source)
  807. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  808. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  809. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  810. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  811. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  812. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1379/1581699.run(Unknown Source)
  813. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  814. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  815. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  816. java.lang.Thread.run(Thread.java:745)
  817.  
  818. ::: {esm00}{Zhv2uiPbSCOlcj7NfdBjpQ}{FX3caSsQSymO6nVuL6zX6g}{10.6.145.100}{10.6.145.100:9300}
  819. Hot threads at 2017-01-31T09:48:30.115Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  820.  
  821. ::: {fe04}{kxzXsbe4R02CtRcFb2dtwQ}{pMcLXYG_SOeEtQ4kQi9pkQ}{10.6.145.197}{10.6.145.197:9300}
  822. Hot threads at 2017-01-31T09:48:30.115Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  823.  
  824. ::: {fe03}{b6L9T7FdSyGjt4ehx-SntQ}{9jU6kUusQMSezOB4XvIRWw}{10.6.145.196}{10.6.145.196:9300}
  825. Hot threads at 2017-01-31T09:48:30.110Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  826.  
  827. 74.1% (370.5ms out of 500ms) cpu usage by thread 'elasticsearch[fe03][warmer][T#5]'
  828. 9/10 snapshots sharing following 18 elements
  829. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  830. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  831. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  832. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  833. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  834. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  835. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  836. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1376/1503916451.load(Unknown Source)
  837. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  838. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  839. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  840. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  841. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  842. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/281979520.run(Unknown Source)
  843. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  844. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  845. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  846. java.lang.Thread.run(Thread.java:745)
  847. unique snapshot
  848. java.util.Arrays.binarySearch0(Arrays.java:1921)
  849. java.util.Arrays.binarySearch(Arrays.java:1864)
  850. org.apache.lucene.util.packed.Packed64SingleBlock.isSupported(Packed64SingleBlock.java:38)
  851. org.apache.lucene.util.packed.PackedInts$Format$2.isSupported(PackedInts.java:118)
  852. org.apache.lucene.util.packed.PackedInts.fastestFormatAndBits(PackedInts.java:264)
  853. org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:938)
  854. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:266)
  855. org.apache.lucene.util.packed.DeltaPackedLongValues$Builder.pack(DeltaPackedLongValues.java:89)
  856. org.apache.lucene.util.packed.MonotonicLongValues$Builder.pack(MonotonicLongValues.java:90)
  857. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:244)
  858. org.apache.lucene.util.packed.PackedLongValues$Builder.add(PackedLongValues.java:227)
  859. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:552)
  860. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  861. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  862. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  863. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  864. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  865. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  866. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1376/1503916451.load(Unknown Source)
  867. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  868. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  869. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  870. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  871. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  872. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/281979520.run(Unknown Source)
  873. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  874. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  875. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  876. java.lang.Thread.run(Thread.java:745)
  877.  
  878. 74.1% (370.5ms out of 500ms) cpu usage by thread 'elasticsearch[fe03][warmer][T#6]'
  879. 4/10 snapshots sharing following 21 elements
  880. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  881. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  882. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  883. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  884. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  885. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  886. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  887. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  888. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  889. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  890. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1376/1503916451.load(Unknown Source)
  891. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  892. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  893. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  894. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  895. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  896. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/281979520.run(Unknown Source)
  897. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  898. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  899. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  900. java.lang.Thread.run(Thread.java:745)
  901. 6/10 snapshots sharing following 17 elements
  902. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  903. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  904. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  905. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  906. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  907. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  908. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1376/1503916451.load(Unknown Source)
  909. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  910. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  911. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  912. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  913. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  914. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1365/281979520.run(Unknown Source)
  915. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  916. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  917. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  918. java.lang.Thread.run(Thread.java:745)
  919.  
  920. ::: {lb02}{Cksl_hyxS-mrz85gHUhwWw}{p-OS8SVtQwa6eY33WhT-zg}{10.6.145.224}{10.6.145.224:9300}
  921. Hot threads at 2017-01-31T09:48:30.116Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  922.  
  923. ::: {lb00}{wzS5bNy5R4G_MzdEUZWL_A}{hcBwo6rQSuiU0JpDACO5zQ}{10.6.145.222}{10.6.145.222:9300}
  924. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  925.  
  926. 0.5% (2.3ms out of 500ms) cpu usage by thread 'elasticsearch[lb00][[transport_server_worker.default]][T#40]'
  927. 7/10 snapshots sharing following 2 elements
  928. io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:873)
  929. java.lang.Thread.run(Thread.java:745)
  930.  
  931. ::: {fe11}{Gwgn-itDThGLJtxC7QAUFA}{BjG4NHAaSSar9j5NafaHow}{10.6.145.204}{10.6.145.204:9300}
  932. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  933.  
  934. 132.3% (661.3ms out of 500ms) cpu usage by thread 'elasticsearch[fe11][warmer][T#6]'
  935. 5/10 snapshots sharing following 20 elements
  936. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  937. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  938. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  939. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  940. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  941. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  942. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  943. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  944. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  945. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  946. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  947. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  948. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  949. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  950. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  951. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  952. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  953. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  954. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  955. java.lang.Thread.run(Thread.java:745)
  956. 3/10 snapshots sharing following 21 elements
  957. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  958. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  959. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  960. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  961. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  962. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  963. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  964. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  965. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  966. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  967. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  968. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  969. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  970. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  971. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  972. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  973. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  974. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  975. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  976. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  977. java.lang.Thread.run(Thread.java:745)
  978. 2/10 snapshots sharing following 18 elements
  979. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  980. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  981. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  982. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  983. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  984. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  985. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  986. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  987. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  988. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  989. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  990. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  991. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  992. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  993. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  994. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  995. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  996. java.lang.Thread.run(Thread.java:745)
  997.  
  998. 114.9% (574.7ms out of 500ms) cpu usage by thread 'elasticsearch[fe11][warmer][T#3]'
  999. 10/10 snapshots sharing following 17 elements
  1000. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1001. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1002. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1003. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1004. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1005. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1006. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  1007. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1008. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1009. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1010. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1011. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1012. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  1013. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1014. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1015. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1016. java.lang.Thread.run(Thread.java:745)
  1017.  
  1018. 45.9% (229.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe11][warmer][T#9]'
  1019. 3/10 snapshots sharing following 21 elements
  1020. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1021. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1022. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1023. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1024. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1025. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1026. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1027. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1028. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1029. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1030. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  1031. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1032. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1033. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1034. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1035. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1036. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  1037. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1038. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1039. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1040. java.lang.Thread.run(Thread.java:745)
  1041. 2/10 snapshots sharing following 21 elements
  1042. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1043. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1044. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1045. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1046. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1047. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1048. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1049. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1050. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1051. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1052. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  1053. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1054. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1055. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1056. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1057. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1058. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  1059. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1060. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1061. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1062. java.lang.Thread.run(Thread.java:745)
  1063. 2/10 snapshots sharing following 17 elements
  1064. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1065. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1066. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1067. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1068. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1069. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1070. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/361349121.load(Unknown Source)
  1071. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1072. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1073. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1074. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1075. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1076. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1629688658.run(Unknown Source)
  1077. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1078. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1079. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1080. java.lang.Thread.run(Thread.java:745)
  1081.  
  1082. ::: {fe17}{43GjqnVJTvG6JysDIxqCIA}{qBs91T_5TvuYkWPI0ObKPA}{10.6.145.210}{10.6.145.210:9300}
  1083. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1084.  
  1085. 113.0% (565ms out of 500ms) cpu usage by thread 'elasticsearch[fe17][warmer][T#3]'
  1086. 3/10 snapshots sharing following 21 elements
  1087. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1088. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1089. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1090. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1091. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1092. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1093. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1094. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1095. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1096. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1097. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1098. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1099. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1100. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1101. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1102. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1103. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1104. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1105. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1106. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1107. java.lang.Thread.run(Thread.java:745)
  1108. 2/10 snapshots sharing following 20 elements
  1109. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1110. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1111. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1112. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1113. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1114. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1115. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1116. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1117. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1118. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1119. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1120. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1121. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1122. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1123. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1124. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1125. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1126. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1127. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1128. java.lang.Thread.run(Thread.java:745)
  1129. 3/10 snapshots sharing following 18 elements
  1130. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  1131. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1132. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1133. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1134. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1135. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1136. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1137. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1138. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1139. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1140. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1141. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1142. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1143. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1144. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1145. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1146. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1147. java.lang.Thread.run(Thread.java:745)
  1148. 2/10 snapshots sharing following 21 elements
  1149. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1150. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1151. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1152. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1153. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1154. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1155. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1156. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1157. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1158. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1159. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1160. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1161. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1162. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1163. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1164. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1165. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1166. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1167. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1168. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1169. java.lang.Thread.run(Thread.java:745)
  1170.  
  1171. 110.0% (549.8ms out of 500ms) cpu usage by thread 'elasticsearch[fe17][warmer][T#8]'
  1172. 5/10 snapshots sharing following 20 elements
  1173. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1174. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1175. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1176. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1177. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1178. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1179. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1180. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1181. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1182. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1183. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1184. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1185. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1186. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1187. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1188. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1189. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1190. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1191. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1192. java.lang.Thread.run(Thread.java:745)
  1193. 2/10 snapshots sharing following 21 elements
  1194. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1195. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1196. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1197. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1198. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1199. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1200. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1201. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1202. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1203. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1204. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1205. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1206. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1207. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1208. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1209. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1210. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1211. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1212. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1213. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1214. java.lang.Thread.run(Thread.java:745)
  1215. 2/10 snapshots sharing following 18 elements
  1216. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  1217. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1218. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1219. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1220. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1221. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1222. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1223. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1224. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1225. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1226. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1227. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1228. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1229. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1230. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1231. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1232. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1233. java.lang.Thread.run(Thread.java:745)
  1234. unique snapshot
  1235. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1236. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1237. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1238. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1239. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1240. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1241. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1242. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1243. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1244. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1245. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1246. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1247. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1248. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1249. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1250. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1251. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1252. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1253. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1254. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1255. java.lang.Thread.run(Thread.java:745)
  1256.  
  1257. 109.0% (544.8ms out of 500ms) cpu usage by thread 'elasticsearch[fe17][warmer][T#7]'
  1258. 4/10 snapshots sharing following 21 elements
  1259. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1260. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1261. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1262. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1263. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1264. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1265. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1266. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1267. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1268. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1269. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1270. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1271. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1272. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1273. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1274. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1275. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1276. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1277. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1278. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1279. java.lang.Thread.run(Thread.java:745)
  1280. 2/10 snapshots sharing following 21 elements
  1281. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  1282. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1283. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1284. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1285. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1286. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1287. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1288. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1289. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1290. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1291. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1292. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1293. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1294. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1295. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1296. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1297. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1298. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1299. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1300. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1301. java.lang.Thread.run(Thread.java:745)
  1302. 3/10 snapshots sharing following 21 elements
  1303. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1304. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1305. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1306. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1307. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1308. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1309. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1310. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1311. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1312. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1313. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1314. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1315. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1316. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1317. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1318. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1319. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1320. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1321. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1322. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1323. java.lang.Thread.run(Thread.java:745)
  1324. unique snapshot
  1325. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  1326. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1327. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1328. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1329. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1330. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1331. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1332. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1392/1927838693.load(Unknown Source)
  1333. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1334. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1335. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1336. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1337. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1338. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1389/132365766.run(Unknown Source)
  1339. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1340. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1341. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1342. java.lang.Thread.run(Thread.java:745)
  1343.  
  1344. ::: {fe16}{BojWjLD9Sty7UHvQiRfQQg}{HEwNQoA5QXqf_CjncEoUlA}{10.6.145.209}{10.6.145.209:9300}
  1345. Hot threads at 2017-01-31T09:48:30.116Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1346.  
  1347. 94.3% (471.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe16][warmer][T#8]'
  1348. 5/10 snapshots sharing following 20 elements
  1349. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1350. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1351. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1352. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1353. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1354. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1355. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1356. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1357. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1358. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1385/1948369692.load(Unknown Source)
  1359. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1360. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1361. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1362. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1363. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1364. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1382/761870325.run(Unknown Source)
  1365. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1366. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1367. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1368. java.lang.Thread.run(Thread.java:745)
  1369. 5/10 snapshots sharing following 21 elements
  1370. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1371. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1372. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1373. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1374. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1375. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1376. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1377. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1378. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1379. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1380. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1385/1948369692.load(Unknown Source)
  1381. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1382. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1383. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1384. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1385. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1386. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1382/761870325.run(Unknown Source)
  1387. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1388. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1389. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1390. java.lang.Thread.run(Thread.java:745)
  1391.  
  1392. 94.3% (471.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe16][warmer][T#5]'
  1393. 10/10 snapshots sharing following 17 elements
  1394. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1395. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1396. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1397. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1398. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1399. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1400. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1385/1948369692.load(Unknown Source)
  1401. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1402. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1403. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1404. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1405. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1406. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1382/761870325.run(Unknown Source)
  1407. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1408. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1409. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1410. java.lang.Thread.run(Thread.java:745)
  1411.  
  1412. ::: {fe10}{-_zl4irVRt6KlvcFsSrdQQ}{xp6WdSlFQG-_jRHXlFNRlg}{10.6.145.203}{10.6.145.203:9300}
  1413. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1414.  
  1415. 108.3% (541.3ms out of 500ms) cpu usage by thread 'elasticsearch[fe10][warmer][T#7]'
  1416. 2/10 snapshots sharing following 21 elements
  1417. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  1418. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1419. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1420. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1421. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1422. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1423. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1424. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1425. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1426. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1427. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1428. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1429. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1430. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1431. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1432. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1433. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1434. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1435. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1436. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1437. java.lang.Thread.run(Thread.java:745)
  1438. 2/10 snapshots sharing following 21 elements
  1439. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1440. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1441. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1442. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1443. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1444. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1445. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1446. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1447. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1448. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1449. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1450. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1451. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1452. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1453. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1454. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1455. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1456. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1457. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1458. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1459. java.lang.Thread.run(Thread.java:745)
  1460. 3/10 snapshots sharing following 21 elements
  1461. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1462. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1463. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1464. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1465. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1466. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1467. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1468. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1469. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1470. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1471. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1472. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1473. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1474. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1475. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1476. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1477. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1478. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1479. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1480. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1481. java.lang.Thread.run(Thread.java:745)
  1482. 3/10 snapshots sharing following 17 elements
  1483. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1484. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1485. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1486. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1487. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1488. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1489. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1490. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1491. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1492. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1493. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1494. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1495. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1496. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1497. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1498. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1499. java.lang.Thread.run(Thread.java:745)
  1500.  
  1501. 107.5% (537.3ms out of 500ms) cpu usage by thread 'elasticsearch[fe10][warmer][T#8]'
  1502. 4/10 snapshots sharing following 21 elements
  1503. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1504. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1505. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1506. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1507. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1508. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1509. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1510. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1511. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1512. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1513. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1514. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1515. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1516. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1517. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1518. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1519. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1520. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1521. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1522. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1523. java.lang.Thread.run(Thread.java:745)
  1524. 3/10 snapshots sharing following 21 elements
  1525. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  1526. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1527. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1528. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1529. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1530. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1531. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1532. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1533. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1534. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1535. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1536. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1537. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1538. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1539. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1540. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1541. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1542. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1543. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1544. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1545. java.lang.Thread.run(Thread.java:745)
  1546. 3/10 snapshots sharing following 17 elements
  1547. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1548. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1549. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1550. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1551. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1552. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1553. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1554. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1555. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1556. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1557. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1558. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1559. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1560. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1561. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1562. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1563. java.lang.Thread.run(Thread.java:745)
  1564.  
  1565. 104.2% (521.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe10][warmer][T#6]'
  1566. 2/10 snapshots sharing following 21 elements
  1567. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1568. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1569. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1570. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1571. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1572. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1573. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1574. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1575. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1576. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1577. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1578. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1579. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1580. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1581. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1582. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1583. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1584. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1585. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1586. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1587. java.lang.Thread.run(Thread.java:745)
  1588. 2/10 snapshots sharing following 21 elements
  1589. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  1590. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1591. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1592. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1593. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1594. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1595. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1596. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1597. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1598. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1599. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1600. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1601. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1602. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1603. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1604. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1605. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1606. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1607. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1608. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1609. java.lang.Thread.run(Thread.java:745)
  1610. 4/10 snapshots sharing following 21 elements
  1611. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1612. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1613. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1614. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1615. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1616. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1617. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1618. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1619. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1620. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1621. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1622. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1623. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1624. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1625. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1626. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1627. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1628. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1629. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1630. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1631. java.lang.Thread.run(Thread.java:745)
  1632. 2/10 snapshots sharing following 17 elements
  1633. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1634. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1635. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1636. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1637. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1638. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1639. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1391/1895458438.load(Unknown Source)
  1640. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1641. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1642. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1643. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1644. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1645. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1388/1717381716.run(Unknown Source)
  1646. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1647. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1648. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1649. java.lang.Thread.run(Thread.java:745)
  1650.  
  1651. ::: {fe02}{WyjeMt3dSLKj1KsIAtWyOg}{y2gyERp9QLiGMmV1YlBnyw}{10.6.145.195}{10.6.145.195:9300}
  1652. Hot threads at 2017-01-31T09:48:30.113Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1653.  
  1654. 114.1% (570.6ms out of 500ms) cpu usage by thread 'elasticsearch[fe02][warmer][T#2]'
  1655. 2/10 snapshots sharing following 21 elements
  1656. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  1657. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  1658. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1659. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1660. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1661. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1662. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1663. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1664. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1665. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1666. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1667. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1668. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1669. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1670. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1671. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1672. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1673. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1674. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1675. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1676. java.lang.Thread.run(Thread.java:745)
  1677. 4/10 snapshots sharing following 21 elements
  1678. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1679. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1680. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1681. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1682. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1683. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1684. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1685. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1686. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1687. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1688. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1689. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1690. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1691. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1692. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1693. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1694. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1695. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1696. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1697. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1698. java.lang.Thread.run(Thread.java:745)
  1699. 3/10 snapshots sharing following 21 elements
  1700. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1701. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1702. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1703. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1704. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1705. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1706. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1707. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1708. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1709. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1710. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1711. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1712. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1713. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1714. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1715. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1716. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1717. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1718. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1719. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1720. java.lang.Thread.run(Thread.java:745)
  1721. unique snapshot
  1722. org.apache.lucene.util.packed.Packed64.set(Packed64.java:223)
  1723. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:268)
  1724. org.apache.lucene.util.packed.DeltaPackedLongValues$Builder.pack(DeltaPackedLongValues.java:89)
  1725. org.apache.lucene.util.packed.MonotonicLongValues$Builder.pack(MonotonicLongValues.java:90)
  1726. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:244)
  1727. org.apache.lucene.util.packed.PackedLongValues$Builder.add(PackedLongValues.java:227)
  1728. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:552)
  1729. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1730. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1731. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1732. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1733. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1734. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1735. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1736. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1737. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1738. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1739. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1740. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1741. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1742. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1743. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1744. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1745. java.lang.Thread.run(Thread.java:745)
  1746.  
  1747. 112.0% (560ms out of 500ms) cpu usage by thread 'elasticsearch[fe02][warmer][T#6]'
  1748. 8/10 snapshots sharing following 18 elements
  1749. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1750. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1751. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1752. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1753. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1754. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1755. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1756. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1757. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1758. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1759. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1760. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1761. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1762. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1763. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1764. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1765. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1766. java.lang.Thread.run(Thread.java:745)
  1767. 2/10 snapshots sharing following 18 elements
  1768. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  1769. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1770. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1771. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1772. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1773. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1774. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1775. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1776. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1777. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1778. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1779. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1780. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1781. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1782. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1783. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1784. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1785. java.lang.Thread.run(Thread.java:745)
  1786.  
  1787. 110.4% (552.1ms out of 500ms) cpu usage by thread 'elasticsearch[fe02][warmer][T#9]'
  1788. 2/10 snapshots sharing following 22 elements
  1789. org.apache.lucene.util.packed.DeltaPackedLongValues$Builder.pack(DeltaPackedLongValues.java:89)
  1790. org.apache.lucene.util.packed.MonotonicLongValues$Builder.pack(MonotonicLongValues.java:90)
  1791. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:244)
  1792. org.apache.lucene.util.packed.PackedLongValues$Builder.add(PackedLongValues.java:227)
  1793. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:552)
  1794. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1795. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1796. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1797. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1798. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1799. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1800. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1801. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1802. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1803. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1804. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1805. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1806. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1807. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1808. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1809. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1810. java.lang.Thread.run(Thread.java:745)
  1811. 4/10 snapshots sharing following 19 elements
  1812. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1813. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1814. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1815. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1816. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1817. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1818. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1819. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1820. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1821. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1822. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1823. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1824. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1825. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1826. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1827. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1828. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1829. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1830. java.lang.Thread.run(Thread.java:745)
  1831. 2/10 snapshots sharing following 18 elements
  1832. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:531)
  1833. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1834. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1835. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1836. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1837. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1838. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1839. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1840. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1841. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1842. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1843. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1844. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1845. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1846. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1847. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1848. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1849. java.lang.Thread.run(Thread.java:745)
  1850. 2/10 snapshots sharing following 17 elements
  1851. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1852. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1853. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1854. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1855. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1856. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1857. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1393/649935546.load(Unknown Source)
  1858. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1859. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1860. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1861. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1862. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1863. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1390/1751521928.run(Unknown Source)
  1864. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1865. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1866. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1867. java.lang.Thread.run(Thread.java:745)
  1868.  
  1869. ::: {fe06}{USXHNcSwSJejHCEGrY89Qw}{H55gacXJT92kHZsAF8ld4w}{10.6.145.199}{10.6.145.199:9300}
  1870. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  1871.  
  1872. 108.6% (542.7ms out of 500ms) cpu usage by thread 'elasticsearch[fe06][warmer][T#8]'
  1873. 3/10 snapshots sharing following 21 elements
  1874. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1875. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1876. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1877. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1878. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1879. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1880. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1881. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1882. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1883. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1884. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1885. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1886. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1887. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1888. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1889. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1890. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1891. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1892. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1893. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1894. java.lang.Thread.run(Thread.java:745)
  1895. 2/10 snapshots sharing following 18 elements
  1896. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  1897. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1898. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1899. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1900. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1901. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1902. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1903. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1904. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1905. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1906. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1907. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1908. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1909. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1910. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1911. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1912. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1913. java.lang.Thread.run(Thread.java:745)
  1914. 3/10 snapshots sharing following 21 elements
  1915. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1916. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1917. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1918. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1919. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1920. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1921. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1922. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1923. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1924. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1925. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1926. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1927. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1928. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1929. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1930. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1931. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1932. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1933. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1934. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1935. java.lang.Thread.run(Thread.java:745)
  1936. 2/10 snapshots sharing following 18 elements
  1937. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1938. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1939. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1940. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1941. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1942. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1943. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1944. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1945. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1946. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1947. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1948. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1949. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1950. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1951. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1952. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1953. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1954. java.lang.Thread.run(Thread.java:745)
  1955.  
  1956. 76.0% (379.9ms out of 500ms) cpu usage by thread 'elasticsearch[fe06][warmer][T#7]'
  1957. 2/10 snapshots sharing following 21 elements
  1958. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  1959. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  1960. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  1961. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1962. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1963. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1964. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1965. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1966. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1967. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1968. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1969. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1970. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1971. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1972. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1973. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1974. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1975. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1976. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1977. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1978. java.lang.Thread.run(Thread.java:745)
  1979. 4/10 snapshots sharing following 21 elements
  1980. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  1981. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  1982. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  1983. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  1984. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  1985. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  1986. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  1987. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  1988. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  1989. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  1990. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  1991. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  1992. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  1993. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  1994. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  1995. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  1996. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  1997. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  1998. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1999. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2000. java.lang.Thread.run(Thread.java:745)
  2001. 3/10 snapshots sharing following 21 elements
  2002. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2003. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2004. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2005. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2006. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2007. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2008. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2009. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2010. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2011. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2012. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2013. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2014. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2015. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2016. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2017. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2018. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2019. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2020. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2021. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2022. java.lang.Thread.run(Thread.java:745)
  2023. unique snapshot
  2024. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  2025. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2026. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2027. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2028. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2029. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2030. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2031. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2032. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2033. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2034. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2035. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2036. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2037. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2038. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2039. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2040. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2041. java.lang.Thread.run(Thread.java:745)
  2042.  
  2043. 75.2% (376.1ms out of 500ms) cpu usage by thread 'elasticsearch[fe06][warmer][T#9]'
  2044. 3/10 snapshots sharing following 21 elements
  2045. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2046. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2047. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2048. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2049. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2050. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2051. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2052. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2053. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2054. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2055. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2056. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2057. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2058. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2059. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2060. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2061. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2062. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2063. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2064. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2065. java.lang.Thread.run(Thread.java:745)
  2066. 4/10 snapshots sharing following 21 elements
  2067. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2068. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2069. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2070. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2071. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2072. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2073. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2074. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2075. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2076. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2077. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2078. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2079. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2080. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2081. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2082. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2083. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2084. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2085. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2086. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2087. java.lang.Thread.run(Thread.java:745)
  2088. 2/10 snapshots sharing following 21 elements
  2089. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  2090. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  2091. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2092. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2093. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2094. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2095. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2096. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2097. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2098. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2099. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2100. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2101. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2102. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2103. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2104. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2105. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2106. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2107. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2108. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2109. java.lang.Thread.run(Thread.java:745)
  2110. unique snapshot
  2111. org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:950)
  2112. org.apache.lucene.util.packed.PackedInts.getMutable(PackedInts.java:939)
  2113. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:266)
  2114. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:244)
  2115. org.apache.lucene.util.packed.PackedLongValues$Builder.add(PackedLongValues.java:227)
  2116. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:551)
  2117. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2118. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2119. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2120. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2121. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2122. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2123. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1390/1039561633.load(Unknown Source)
  2124. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2125. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2126. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2127. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2128. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2129. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1387/208534291.run(Unknown Source)
  2130. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2131. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2132. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2133. java.lang.Thread.run(Thread.java:745)
  2134.  
  2135. ::: {fe01}{GauHId3tSY6H8_UAKCFWhQ}{1k4zSgYGSzGuhdY9Qy534A}{10.6.145.194}{10.6.145.194:9300}
  2136. Hot threads at 2017-01-31T09:48:30.116Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2137.  
  2138. 116.4% (582ms out of 500ms) cpu usage by thread 'elasticsearch[fe01][warmer][T#7]'
  2139. 8/10 snapshots sharing following 18 elements
  2140. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2141. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2142. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2143. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2144. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2145. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2146. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2147. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2148. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2149. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2150. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2151. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2152. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2153. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2154. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2155. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2156. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2157. java.lang.Thread.run(Thread.java:745)
  2158. 2/10 snapshots sharing following 18 elements
  2159. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  2160. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2161. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2162. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2163. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2164. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2165. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2166. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2167. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2168. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2169. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2170. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2171. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2172. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2173. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2174. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2175. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2176. java.lang.Thread.run(Thread.java:745)
  2177.  
  2178. 113.3% (566.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe01][warmer][T#9]'
  2179. 4/10 snapshots sharing following 21 elements
  2180. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2181. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2182. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2183. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2184. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2185. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2186. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2187. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2188. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2189. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2190. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2191. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2192. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2193. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2194. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2195. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2196. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2197. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2198. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2199. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2200. java.lang.Thread.run(Thread.java:745)
  2201. 3/10 snapshots sharing following 21 elements
  2202. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  2203. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  2204. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2205. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2206. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2207. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2208. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2209. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2210. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2211. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2212. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2213. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2214. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2215. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2216. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2217. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2218. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2219. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2220. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2221. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2222. java.lang.Thread.run(Thread.java:745)
  2223. 3/10 snapshots sharing following 21 elements
  2224. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2225. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2226. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2227. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2228. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2229. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2230. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2231. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2232. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2233. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2234. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2235. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2236. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2237. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2238. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2239. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2240. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2241. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2242. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2243. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2244. java.lang.Thread.run(Thread.java:745)
  2245.  
  2246. 113.3% (566.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe01][warmer][T#6]'
  2247. 5/10 snapshots sharing following 21 elements
  2248. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2249. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2250. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2251. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2252. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2253. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2254. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2255. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2256. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2257. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2258. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2259. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2260. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2261. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2262. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2263. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2264. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2265. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2266. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2267. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2268. java.lang.Thread.run(Thread.java:745)
  2269. 4/10 snapshots sharing following 18 elements
  2270. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2271. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2272. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2273. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2274. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2275. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2276. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2277. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2278. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2279. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2280. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2281. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2282. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2283. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2284. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2285. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2286. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2287. java.lang.Thread.run(Thread.java:745)
  2288. unique snapshot
  2289. org.apache.lucene.util.packed.Packed64.set(Packed64.java:223)
  2290. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:268)
  2291. org.apache.lucene.util.packed.DeltaPackedLongValues$Builder.pack(DeltaPackedLongValues.java:89)
  2292. org.apache.lucene.util.packed.MonotonicLongValues$Builder.pack(MonotonicLongValues.java:90)
  2293. org.apache.lucene.util.packed.PackedLongValues$Builder.pack(PackedLongValues.java:244)
  2294. org.apache.lucene.util.packed.PackedLongValues$Builder.add(PackedLongValues.java:227)
  2295. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:552)
  2296. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2297. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2298. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2299. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2300. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2301. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2302. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1023239122.load(Unknown Source)
  2303. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2304. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2305. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2306. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2307. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2308. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/1926206533.run(Unknown Source)
  2309. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2310. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2311. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2312. java.lang.Thread.run(Thread.java:745)
  2313.  
  2314. ::: {fe15}{uZ2ldzlxRhW15UKpixDqpw}{XJ4FWRIATZqNcedfffXgYg}{10.6.145.208}{10.6.145.208:9300}
  2315. Hot threads at 2017-01-31T09:48:30.112Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2316.  
  2317. 104.9% (524.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe15][warmer][T#8]'
  2318. 3/10 snapshots sharing following 21 elements
  2319. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2320. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2321. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2322. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2323. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2324. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2325. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2326. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2327. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2328. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2329. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2330. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2331. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2332. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2333. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2334. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2335. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2336. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2337. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2338. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2339. java.lang.Thread.run(Thread.java:745)
  2340. 5/10 snapshots sharing following 19 elements
  2341. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2342. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2343. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2344. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2345. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2346. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2347. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2348. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2349. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2350. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2351. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2352. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2353. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2354. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2355. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2356. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2357. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2358. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2359. java.lang.Thread.run(Thread.java:745)
  2360. 2/10 snapshots sharing following 17 elements
  2361. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2362. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2363. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2364. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2365. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2366. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2367. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2368. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2369. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2370. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2371. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2372. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2373. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2374. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2375. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2376. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2377. java.lang.Thread.run(Thread.java:745)
  2378.  
  2379. 104.7% (523.5ms out of 500ms) cpu usage by thread 'elasticsearch[fe15][warmer][T#4]'
  2380. 3/10 snapshots sharing following 21 elements
  2381. org.apache.lucene.codecs.lucene54.Lucene54DocValuesProducer$CompressedBinaryDocValues$CompressedBinaryTermsEnum.next(Lucene54DocValuesProducer.java:1365)
  2382. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2383. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2384. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2385. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2386. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2387. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2388. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2389. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2390. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2391. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2392. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2393. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2394. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2395. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2396. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2397. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2398. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2399. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2400. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2401. java.lang.Thread.run(Thread.java:745)
  2402. 2/10 snapshots sharing following 21 elements
  2403. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2404. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2405. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2406. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2407. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2408. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2409. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2410. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2411. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2412. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2413. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2414. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2415. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2416. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2417. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2418. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2419. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2420. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2421. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2422. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2423. java.lang.Thread.run(Thread.java:745)
  2424. 3/10 snapshots sharing following 20 elements
  2425. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:273)
  2426. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2427. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2428. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2429. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2430. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2431. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2432. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2433. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2434. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2435. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2436. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2437. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2438. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2439. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2440. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2441. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2442. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2443. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2444. java.lang.Thread.run(Thread.java:745)
  2445. 2/10 snapshots sharing following 2 elements
  2446. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2447. java.lang.Thread.run(Thread.java:745)
  2448.  
  2449. 103.1% (515.4ms out of 500ms) cpu usage by thread 'elasticsearch[fe15][warmer][T#6]'
  2450. 5/10 snapshots sharing following 20 elements
  2451. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:273)
  2452. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2453. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2454. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2455. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2456. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2457. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2458. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2459. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2460. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2461. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2462. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2463. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2464. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2465. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2466. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2467. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2468. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2469. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2470. java.lang.Thread.run(Thread.java:745)
  2471. 5/10 snapshots sharing following 17 elements
  2472. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2473. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2474. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2475. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2476. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2477. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2478. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1453/1760224786.load(Unknown Source)
  2479. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2480. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2481. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2482. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2483. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2484. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1450/863498755.run(Unknown Source)
  2485. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2486. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2487. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2488. java.lang.Thread.run(Thread.java:745)
  2489.  
  2490. ::: {fe08}{sPrnWdqBTzSnhdErrn_SCg}{aE9UQv0sTSiWT45S-wvD1Q}{10.6.145.201}{10.6.145.201:9300}
  2491. Hot threads at 2017-01-31T09:48:30.117Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2492.  
  2493. 113.4% (566.8ms out of 500ms) cpu usage by thread 'elasticsearch[fe08][warmer][T#5]'
  2494. 3/10 snapshots sharing following 21 elements
  2495. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2496. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2497. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2498. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2499. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2500. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2501. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2502. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2503. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2504. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2505. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2506. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2507. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2508. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2509. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2510. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2511. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2512. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2513. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2514. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2515. java.lang.Thread.run(Thread.java:745)
  2516. 7/10 snapshots sharing following 17 elements
  2517. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2518. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2519. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2520. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2521. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2522. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2523. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2524. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2525. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2526. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2527. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2528. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2529. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2530. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2531. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2532. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2533. java.lang.Thread.run(Thread.java:745)
  2534.  
  2535. 108.1% (540.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe08][warmer][T#7]'
  2536. 2/10 snapshots sharing following 18 elements
  2537. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:546)
  2538. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2539. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2540. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2541. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2542. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2543. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2544. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2545. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2546. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2547. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2548. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2549. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2550. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2551. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2552. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2553. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2554. java.lang.Thread.run(Thread.java:745)
  2555. 4/10 snapshots sharing following 21 elements
  2556. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2557. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2558. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2559. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2560. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2561. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2562. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2563. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2564. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2565. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2566. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2567. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2568. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2569. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2570. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2571. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2572. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2573. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2574. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2575. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2576. java.lang.Thread.run(Thread.java:745)
  2577. 4/10 snapshots sharing following 17 elements
  2578. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2579. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2580. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2581. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2582. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2583. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2584. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2585. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2586. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2587. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2588. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2589. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2590. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2591. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2592. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2593. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2594. java.lang.Thread.run(Thread.java:745)
  2595.  
  2596. 108.0% (540.2ms out of 500ms) cpu usage by thread 'elasticsearch[fe08][warmer][T#9]'
  2597. 4/10 snapshots sharing following 21 elements
  2598. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  2599. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  2600. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2601. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2602. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2603. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2604. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2605. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2606. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2607. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2608. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2609. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2610. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2611. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2612. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2613. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2614. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2615. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2616. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2617. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2618. java.lang.Thread.run(Thread.java:745)
  2619. 2/10 snapshots sharing following 20 elements
  2620. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
  2621. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2622. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2623. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2624. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2625. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2626. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2627. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2628. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2629. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2630. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2631. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2632. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2633. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2634. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2635. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2636. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2637. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2638. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2639. java.lang.Thread.run(Thread.java:745)
  2640. 4/10 snapshots sharing following 21 elements
  2641. org.apache.lucene.index.MultiTermsEnum$TermMergeQueue.fillTop(MultiTermsEnum.java:429)
  2642. org.apache.lucene.index.MultiTermsEnum.pullTop(MultiTermsEnum.java:267)
  2643. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:305)
  2644. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2645. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2646. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2647. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2648. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2649. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2650. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2651. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/1996990314.load(Unknown Source)
  2652. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2653. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2654. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2655. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2656. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2657. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/659771061.run(Unknown Source)
  2658. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2659. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2660. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2661. java.lang.Thread.run(Thread.java:745)
  2662.  
  2663. ::: {esm01}{ebJtz8zKQUm4sSq1fPBMXQ}{faXQmM7GRrCSXQqMCeVcYQ}{10.6.145.101}{10.6.145.101:9300}
  2664. Hot threads at 2017-01-31T09:48:30.113Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2665.  
  2666. ::: {fe07}{q8Wx-VnqR-2oeqHKuxbsFw}{zCnOgv9gQCuPR_eenAodXQ}{10.6.145.200}{10.6.145.200:9300}
  2667. Hot threads at 2017-01-31T09:48:30.118Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2668.  
  2669. ::: {fe19}{g6XG4QsWSCStzNEl-J6XBQ}{-Yim-lisR9StgadDuAb_wQ}{10.6.145.212}{10.6.145.212:9300}
  2670. Hot threads at 2017-01-31T09:48:30.116Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  2671.  
  2672. 106.2% (531.1ms out of 500ms) cpu usage by thread 'elasticsearch[fe19][warmer][T#1]'
  2673. 5/10 snapshots sharing following 21 elements
  2674. org.apache.lucene.util.PriorityQueue.updateTop(PriorityQueue.java:211)
  2675. org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:279)
  2676. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2677. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2678. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2679. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2680. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2681. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2682. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2683. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2684. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/70753043.load(Unknown Source)
  2685. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2686. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2687. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2688. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2689. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2690. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/43031635.run(Unknown Source)
  2691. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2692. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2693. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2694. java.lang.Thread.run(Thread.java:745)
  2695. 5/10 snapshots sharing following 17 elements
  2696. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2697. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2698. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2699. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2700. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2701. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2702. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/70753043.load(Unknown Source)
  2703. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2704. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2705. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2706. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2707. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2708. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/43031635.run(Unknown Source)
  2709. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2710. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2711. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2712. java.lang.Thread.run(Thread.java:745)
  2713.  
  2714. 104.7% (523.6ms out of 500ms) cpu usage by thread 'elasticsearch[fe19][warmer][T#9]'
  2715. 3/10 snapshots sharing following 19 elements
  2716. org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
  2717. org.apache.lucene.index.MultiDocValues$OrdinalMap.<init>(MultiDocValues.java:527)
  2718. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2719. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2720. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2721. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2722. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2723. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2724. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/70753043.load(Unknown Source)
  2725. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2726. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2727. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2728. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2729. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2730. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/43031635.run(Unknown Source)
  2731. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2732. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2733. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2734. java.lang.Thread.run(Thread.java:745)
  2735. 7/10 snapshots sharing following 17 elements
  2736. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2737. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2738. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2739. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2740. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2741. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2742. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/70753043.load(Unknown Source)
  2743. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2744. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2745. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2746. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2747. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2748. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/43031635.run(Unknown Source)
  2749. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2750. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2751. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2752. java.lang.Thread.run(Thread.java:745)
  2753.  
  2754. 103.4% (516.9ms out of 500ms) cpu usage by thread 'elasticsearch[fe19][warmer][T#7]'
  2755. 10/10 snapshots sharing following 17 elements
  2756. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:484)
  2757. org.apache.lucene.index.MultiDocValues$OrdinalMap.build(MultiDocValues.java:448)
  2758. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.buildOrdinalMap(ParentChildIndexFieldData.java:175)
  2759. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:199)
  2760. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.localGlobalDirect(ParentChildIndexFieldData.java:71)
  2761. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:159)
  2762. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$1383/70753043.load(Unknown Source)
  2763. org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:398)
  2764. org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:154)
  2765. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:160)
  2766. org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData.loadGlobal(ParentChildIndexFieldData.java:71)
  2767. org.elasticsearch.index.IndexWarmer$FieldDataWarmer.lambda$warmReader$1(IndexWarmer.java:142)
  2768. org.elasticsearch.index.IndexWarmer$FieldDataWarmer$$Lambda$1380/43031635.run(Unknown Source)
  2769. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:458)
  2770. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  2771. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  2772. java.lang.Thread.run(Thread.java:745)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!