API tools faq
paste
Advertisement
Guest User

Untitled

a guest
May 11th, 2017
96
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 35.84 KB | None | 0 0
raw download clone embed print report
  1.  
  2. > ::: {es-2}{x3-HtrkZRv2TbmluIl8A5Q}{XbAMW1_IQBCyOf-agGJuDA}{ip}{ip:9300}
  3. > Hot threads at 2017-05-11T15:20:32.077Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  4. >
  5. > 18.0% (90.1ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#4]'
  6. > 2/10 snapshots sharing following 5 elements
  7. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  8. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  9. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  10. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  11. > java.lang.Thread.run(Thread.java:745)
  12. > 4/10 snapshots sharing following 10 elements
  13. > sun.misc.Unsafe.park(Native Method)
  14. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  15. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  16. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  17. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  18. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  19. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  20. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  21. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  22. > java.lang.Thread.run(Thread.java:745)
  23. >
  24. > 15.1% (75.6ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#1]'
  25. > 2/10 snapshots sharing following 25 elements
  26. > org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:143)
  27. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:113)
  28. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:69)
  29. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:939)
  30. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:908)
  31. > org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:113)
  32. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322)
  33. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264)
  34. > org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888)
  35. > org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885)
  36. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  37. > org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1658)
  38. > org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897)
  39. > org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93)
  40. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281)
  41. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  42. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260)
  43. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252)
  44. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  45. > org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:627)
  46. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  47. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  48. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  49. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  50. > java.lang.Thread.run(Thread.java:745)
  51. > 4/10 snapshots sharing following 10 elements
  52. > sun.misc.Unsafe.park(Native Method)
  53. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  54. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  55. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  56. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  57. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  58. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  59. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  60. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  61. > java.lang.Thread.run(Thread.java:745)
  62. >
  63. > 14.3% (71.6ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#2]'
  64. > 4/10 snapshots sharing following 16 elements
  65. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  66. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  67. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  68. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  69. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  70. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  71. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  72. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  73. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  74. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  75. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  76. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  77. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  78. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  79. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  80. > java.lang.Thread.run(Thread.java:745)
  81. > 5/10 snapshots sharing following 10 elements
  82. > sun.misc.Unsafe.park(Native Method)
  83. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  84. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  85. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  86. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  87. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  88. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  89. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  90. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  91. > java.lang.Thread.run(Thread.java:745)
  92.  
  93. > ::: {elk}{cUzRnyEvRae4M8pyO67PgA}{sb0uq08STgunIYU2HGFRGQ}{ip}{ip:9300}
  94. > Hot threads at 2017-05-11T15:20:33.501Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  95. >
  96. > 0.0% (70.4micros out of 500ms) cpu usage by thread 'elasticsearch[elk][[timer]]'
  97. > 10/10 snapshots sharing following 2 elements
  98. > java.lang.Thread.sleep(Native Method)
  99. > org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:536)
  100.  
  101. > ::: {es-3}{1wYo1ev5RfqdVtEWMi3kCA}{syeNlKqMRUevfcq4NeboMA}{ip}{ip:9300}
  102. > Hot threads at 2017-05-11T15:20:32.075Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  103. >
  104. > 18.3% (91.5ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#1]'
  105. > 3/10 snapshots sharing following 16 elements
  106. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  107. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  108. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  109. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  110. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  111. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  112. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  113. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  114. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  115. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  116. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  117. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  118. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  119. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  120. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  121. > java.lang.Thread.run(Thread.java:745)
  122. > 2/10 snapshots sharing following 10 elements
  123. > sun.misc.Unsafe.park(Native Method)
  124. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  125. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  126. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  127. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  128. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  129. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  130. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  131. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  132. > java.lang.Thread.run(Thread.java:745)
  133. >
  134. > 17.0% (84.8ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#3]'
  135. > 2/10 snapshots sharing following 30 elements
  136. > sun.nio.ch.FileDispatcherImpl.force0(Native Method)
  137. > sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:76)
  138. > sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388)
  139. > org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:272)
  140. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
  141. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
  142. > org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
  143. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
  144. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
  145. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
  146. > org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
  147. > org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
  148. > org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
  149. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
  150. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  151. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  152. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  153. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  154. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  155. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  156. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  157. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  158. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  159. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  160. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  161. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  162. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  163. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  164. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  165. > java.lang.Thread.run(Thread.java:745)
  166. > 4/10 snapshots sharing following 16 elements
  167. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  168. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  169. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  170. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  171. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  172. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  173. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  174. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  175. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  176. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  177. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  178. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  179. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  180. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  181. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  182. > java.lang.Thread.run(Thread.java:745)
  183. > 3/10 snapshots sharing following 10 elements
  184. > sun.misc.Unsafe.park(Native Method)
  185. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  186. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  187. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  188. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  189. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  190. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  191. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  192. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  193. > java.lang.Thread.run(Thread.java:745)
  194. > unique snapshot
  195. > sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
  196. > sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
  197. > sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
  198. > io.netty.channel.nio.NioEventLoop.wakeup(NioEventLoop.java:687)
  199. > io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:753)
  200. > io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:1007)
  201. > io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:825)
  202. > io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:794)
  203. > io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:831)
  204. > io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1032)
  205. > io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:296)
  206. > org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:411)
  207. > org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:94)
  208. > org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1089)
  209. > org.elasticsearch.transport.TcpTransport.sendResponse(TcpTransport.java:1164)
  210. > org.elasticsearch.transport.TcpTransport.sendResponse(TcpTransport.java:1132)
  211. > org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:67)
  212. > org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61)
  213. > org.elasticsearch.transport.DelegatingTransportChannel.sendResponse(DelegatingTransportChannel.java:60)
  214. > org.elasticsearch.transport.RequestHandlerRegistry$TransportChannelWrapper.sendResponse(RequestHandlerRegistry.java:111)
  215. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$ResponseListener.onResponse(TransportReplicationAction.java:581)
  216. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$ResponseListener.onResponse(TransportReplicationAction.java:572)
  217. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaResult.respond(TransportReplicationAction.java:434)
  218. > org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.respondIfPossible(TransportWriteAction.java:179)
  219. > org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.respond(TransportWriteAction.java:170)
  220. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:496)
  221. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  222. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  223. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  224. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  225. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  226. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  227. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  228. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  229. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  230. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  231. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  232. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  233. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  234. > java.lang.Thread.run(Thread.java:745)
  235. >
  236. > 14.1% (70.4ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#4]'
  237. > 2/10 snapshots sharing following 30 elements
  238. > sun.nio.ch.FileDispatcherImpl.force0(Native Method)
  239. > sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:76)
  240. > sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388)
  241. > org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:272)
  242. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
  243. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
  244. > org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
  245. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
  246. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
  247. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
  248. > org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
  249. > org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
  250. > org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
  251. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
  252. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  253. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  254. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  255. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  256. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  257. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  258. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  259. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  260. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  261. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  262. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  263. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  264. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  265. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  266. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  267. > java.lang.Thread.run(Thread.java:745)
  268. > 6/10 snapshots sharing following 10 elements
  269. > sun.misc.Unsafe.park(Native Method)
  270. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  271. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  272. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  273. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  274. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  275. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  276. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  277. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  278. > java.lang.Thread.run(Thread.java:745)
  279. > 2/10 snapshots sharing following 5 elements
  280. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  281. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  282. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  283. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  284. > java.lang.Thread.run(Thread.java:745)
  285.  
  286. > ::: {es-1}{-NtWQxXcT8SyJ7bT6tcMfQ}{yQIPqr1ZTYCoaACRyhRzvQ}{ip}{ip:9300}
  287. > Hot threads at 2017-05-11T15:20:32.093Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  288. >
  289. > 19.8% (99.2ms out of 500ms) cpu usage by thread 'elasticsearch[es-1][bulk][T#2]'
  290. > 2/10 snapshots sharing following 26 elements
  291. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
  292. > org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
  293. > org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
  294. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
  295. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
  296. > org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
  297. > org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
  298. > org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
  299. > org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
  300. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
  301. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  302. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  303. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  304. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  305. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  306. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  307. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  308. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  309. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  310. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  311. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  312. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  313. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  314. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  315. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  316. > java.lang.Thread.run(Thread.java:745)
  317. > 3/10 snapshots sharing following 10 elements
  318. > sun.misc.Unsafe.park(Native Method)
  319. > java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  320. > java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
  321. > java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  322. > java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
  323. > org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
  324. > java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  325. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  326. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  327. > java.lang.Thread.run(Thread.java:745)
  328. > 2/10 snapshots sharing following 17 elements
  329. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:376)
  330. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
  331. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
  332. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
  333. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  334. > org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
  335. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
  336. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  337. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
  338. > org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
  339. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  340. > org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
  341. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  342. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  343. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  344. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  345. > java.lang.Thread.run(Thread.java:745)
  346. > 2/10 snapshots sharing following 25 elements
  347. > org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:143)
  348. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:113)
  349. > org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:69)
  350. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:939)
  351. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:908)
  352. > org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:113)
  353. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322)
  354. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264)
  355. > org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888)
  356. > org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885)
  357. > org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
  358. > org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1658)
  359. > org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897)
  360. > org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93)
  361. > org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281)
  362. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  363. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260)
  364. > org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252)
  365. > org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
  366. > org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:627)
  367. > org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
  368. > org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  369. > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  370. > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  371. > java.lang.Thread.run(Thread.java:745)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!