API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Apr 25th, 2019
148
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 77.54 KB | None | 0 0
raw download clone embed print report
  1. ::: {elastic-data-2}{p5ZZL7h0SsO8_2HmyXEw4w}{lcH__OpLRFmQ82rhFu8eCQ}{10.10.10.38}{10.10.10.38:9300}{ml.machine_memory=67554250752, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
  2. Hot threads at 2019-04-25T09:52:18.754, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
  3.  
  4. 0.0% (55.7micros out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-2][[timer]]'
  5. 10/10 snapshots sharing following 2 elements
  6. java.lang.Thread.sleep(Native Method)
  7. org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:543)
  8.  
  9. 0.0% (27micros out of 500ms) cpu usage by thread 'threadDeathWatcher-2-1'
  10. 10/10 snapshots sharing following 4 elements
  11. java.lang.Thread.sleep(Native Method)
  12. io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
  13. io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
  14. java.lang.Thread.run(Thread.java:748)
  15.  
  16. 0.0% (17.8micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
  17. 10/10 snapshots sharing following 2 elements
  18. java.lang.Thread.sleep(Native Method)
  19. org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:161)
  20.  
  21. 0.0% (0s out of 500ms) cpu usage by thread 'DestroyJavaVM'
  22. unique snapshot
  23. unique snapshot
  24. unique snapshot
  25. unique snapshot
  26. unique snapshot
  27. unique snapshot
  28. unique snapshot
  29. unique snapshot
  30. unique snapshot
  31. unique snapshot
  32.  
  33. 0.0% (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
  34. 10/10 snapshots sharing following 6 elements
  35. java.io.FileInputStream.readBytes(Native Method)
  36. java.io.FileInputStream.read(FileInputStream.java:233)
  37. org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105)
  38. org.elasticsearch.xpack.ml.job.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:64)
  39. org.elasticsearch.xpack.ml.job.process.NativeController$$Lambda$1707/2009502388.run(Unknown Source)
  40. java.lang.Thread.run(Thread.java:748)
  41.  
  42. 0.0% (0s out of 500ms) cpu usage by thread 'process reaper'
  43. 10/10 snapshots sharing following 6 elements
  44. java.lang.UNIXProcess.waitForProcessExit(Native Method)
  45. java.lang.UNIXProcess.lambda$initStreams$3(UNIXProcess.java:289)
  46. java.lang.UNIXProcess$$Lambda$125/270056930.run(Unknown Source)
  47. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  48. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  49. java.lang.Thread.run(Thread.java:748)
  50.  
  51. 0.0% (0s out of 500ms) cpu usage by thread 'Connection evictor'
  52. 10/10 snapshots sharing following 3 elements
  53. java.lang.Thread.sleep(Native Method)
  54. org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
  55. java.lang.Thread.run(Thread.java:748)
  56.  
  57. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/6.4.3]'
  58. 10/10 snapshots sharing following 8 elements
  59. sun.misc.Unsafe.park(Native Method)
  60. java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  61. java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  62. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  63. java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  64. java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
  65. org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:82)
  66. java.lang.Thread.run(Thread.java:748)
  67.  
  68. ::: {elastic-data-1}{3BrlTlQySAGpSG2EAzORyg}{DiQ4SAVTRuaQnY-Se4b-VQ}{10.10.10.39}{10.10.10.39:9300}{ml.machine_memory=67554172928, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
  69. Hot threads at 2019-04-25T09:52:18.754, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
  70.  
  71. 0.0% (49.9micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
  72. 10/10 snapshots sharing following 2 elements
  73. java.lang.Thread.sleep(Native Method)
  74. org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:161)
  75.  
  76. 0.0% (44.5micros out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][[timer]]'
  77. 10/10 snapshots sharing following 2 elements
  78. java.lang.Thread.sleep(Native Method)
  79. org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:543)
  80.  
  81. 0.0% (0s out of 500ms) cpu usage by thread 'threadDeathWatcher-2-1'
  82. 10/10 snapshots sharing following 4 elements
  83. java.lang.Thread.sleep(Native Method)
  84. io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
  85. io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
  86. java.lang.Thread.run(Thread.java:748)
  87.  
  88. 0.0% (0s out of 500ms) cpu usage by thread 'DestroyJavaVM'
  89. unique snapshot
  90. unique snapshot
  91. unique snapshot
  92. unique snapshot
  93. unique snapshot
  94. unique snapshot
  95. unique snapshot
  96. unique snapshot
  97. unique snapshot
  98. unique snapshot
  99.  
  100. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][generic][T#2]'
  101. 10/10 snapshots sharing following 21 elements
  102. sun.misc.Unsafe.park(Native Method)
  103. java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  104. java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  105. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  106. java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  107. org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:251)
  108. org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:94)
  109. org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:44)
  110. org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:32)
  111. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$1(PeerRecoveryTargetService.java:200)
  112. org.elasticsearch.indices.recovery.PeerRecoveryTargetService$$Lambda$2828/349215753.run(Unknown Source)
  113. org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105)
  114. org.elasticsearch.common.util.CancellableThreads.execute(CancellableThreads.java:86)
  115. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.doRecovery(PeerRecoveryTargetService.java:193)
  116. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.access$900(PeerRecoveryTargetService.java:80)
  117. org.elasticsearch.indices.recovery.PeerRecoveryTargetService$RecoveryRunner.doRun(PeerRecoveryTargetService.java:623)
  118. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  119. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  120. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  121. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  122. java.lang.Thread.run(Thread.java:748)
  123.  
  124. 0.0% (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
  125. 10/10 snapshots sharing following 6 elements
  126. java.io.FileInputStream.readBytes(Native Method)
  127. java.io.FileInputStream.read(FileInputStream.java:233)
  128. org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105)
  129. org.elasticsearch.xpack.ml.job.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:64)
  130. org.elasticsearch.xpack.ml.job.process.NativeController$$Lambda$1707/2009502388.run(Unknown Source)
  131. java.lang.Thread.run(Thread.java:748)
  132.  
  133. 0.0% (0s out of 500ms) cpu usage by thread 'Connection evictor'
  134. 10/10 snapshots sharing following 3 elements
  135. java.lang.Thread.sleep(Native Method)
  136. org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
  137. java.lang.Thread.run(Thread.java:748)
  138.  
  139. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][generic][T#1]'
  140. 10/10 snapshots sharing following 21 elements
  141. sun.misc.Unsafe.park(Native Method)
  142. java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  143. java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  144. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  145. java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  146. org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:251)
  147. org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:94)
  148. org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:44)
  149. org.elasticsearch.transport.PlainTransportFuture.txGet(PlainTransportFuture.java:32)
  150. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$1(PeerRecoveryTargetService.java:200)
  151. org.elasticsearch.indices.recovery.PeerRecoveryTargetService$$Lambda$2828/349215753.run(Unknown Source)
  152. org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105)
  153. org.elasticsearch.common.util.CancellableThreads.execute(CancellableThreads.java:86)
  154. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.doRecovery(PeerRecoveryTargetService.java:193)
  155. org.elasticsearch.indices.recovery.PeerRecoveryTargetService.access$900(PeerRecoveryTargetService.java:80)
  156. org.elasticsearch.indices.recovery.PeerRecoveryTargetService$RecoveryRunner.doRun(PeerRecoveryTargetService.java:623)
  157. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  158. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  159. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  160. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  161. java.lang.Thread.run(Thread.java:748)
  162.  
  163. 0.0% (0s out of 500ms) cpu usage by thread 'process reaper'
  164. 10/10 snapshots sharing following 6 elements
  165. java.lang.UNIXProcess.waitForProcessExit(Native Method)
  166. java.lang.UNIXProcess.lambda$initStreams$3(UNIXProcess.java:289)
  167. java.lang.UNIXProcess$$Lambda$125/270056930.run(Unknown Source)
  168. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  169. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  170. java.lang.Thread.run(Thread.java:748)
  171.  
  172. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/6.4.3]'
  173. 10/10 snapshots sharing following 8 elements
  174. sun.misc.Unsafe.park(Native Method)
  175. java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  176. java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  177. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  178. java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  179. java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
  180. org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:82)
  181. java.lang.Thread.run(Thread.java:748)
  182.  
  183. ::: {elastic-data-0}{0McTAmanQsiWYKNJpVYC-A}{mX2KdpDmSY6EmV38ZOESWQ}{10.10.10.40}{10.10.10.40:9300}{ml.machine_memory=67554168832, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}
  184. Hot threads at 2019-04-25T09:52:18.754, interval=500ms, busiestThreads=9999, ignoreIdleThreads=true:
  185.  
  186. 40.8% (204ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#5]'
  187. 2/10 snapshots sharing following 27 elements
  188. java.util.TreeMap.getEntry(TreeMap.java:359)
  189. java.util.TreeMap.get(TreeMap.java:278)
  190. org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:315)
  191. org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
  192. org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
  193. org.elasticsearch.search.suggest.completion.CompletionFieldStats.completionStats(CompletionFieldStats.java:54)
  194. org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1005)
  195. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:210)
  196. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  197. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  198. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  199. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  200. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  201. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  202. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  203. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  204. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  205. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  206. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  207. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  208. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  209. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1605)
  210. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  211. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  212. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  213. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  214. java.lang.Thread.run(Thread.java:748)
  215. 8/10 snapshots sharing following 19 elements
  216. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  217. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  218. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  219. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  220. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  221. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  222. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  223. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  224. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  225. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  226. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  227. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  228. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  229. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1605)
  230. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  231. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  232. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  233. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  234. java.lang.Thread.run(Thread.java:748)
  235.  
  236. 40.4% (202.1ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#2]'
  237. 3/10 snapshots sharing following 41 elements
  238. java.io.UnixFileSystem.canonicalize0(Native Method)
  239. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  240. java.io.File.getCanonicalPath(File.java:618)
  241. java.io.FilePermission$1.run(FilePermission.java:215)
  242. java.io.FilePermission$1.run(FilePermission.java:203)
  243. java.security.AccessController.doPrivileged(Native Method)
  244. java.io.FilePermission.init(FilePermission.java:203)
  245. java.io.FilePermission.<init>(FilePermission.java:277)
  246. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  247. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  248. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  249. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  250. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  251. java.nio.file.Files.readAttributes(Files.java:1737)
  252. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  253. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:139)
  254. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:414)
  255. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:402)
  256. org.elasticsearch.index.translog.Translog.stats(Translog.java:833)
  257. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:483)
  258. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:999)
  259. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  260. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  261. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  262. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  263. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  264. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  265. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  266. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  267. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  268. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  269. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  270. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  271. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  272. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  273. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  274. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  275. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  276. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  277. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  278. java.lang.Thread.run(Thread.java:748)
  279. 2/10 snapshots sharing following 27 elements
  280. java.util.TreeMap.getEntry(TreeMap.java:359)
  281. java.util.TreeMap.get(TreeMap.java:278)
  282. org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.terms(PerFieldPostingsFormat.java:315)
  283. org.apache.lucene.index.CodecReader.terms(CodecReader.java:106)
  284. org.apache.lucene.index.FilterLeafReader.terms(FilterLeafReader.java:352)
  285. org.elasticsearch.search.suggest.completion.CompletionFieldStats.completionStats(CompletionFieldStats.java:54)
  286. org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1005)
  287. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:210)
  288. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  289. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  290. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  291. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  292. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  293. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  294. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  295. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  296. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  297. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  298. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  299. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  300. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  301. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  302. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  303. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  304. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  305. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  306. java.lang.Thread.run(Thread.java:748)
  307. 5/10 snapshots sharing following 45 elements
  308. java.io.UnixFileSystem.canonicalize0(Native Method)
  309. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  310. java.io.File.getCanonicalPath(File.java:618)
  311. java.io.FilePermission$1.run(FilePermission.java:215)
  312. java.io.FilePermission$1.run(FilePermission.java:203)
  313. java.security.AccessController.doPrivileged(Native Method)
  314. java.io.FilePermission.init(FilePermission.java:203)
  315. java.io.FilePermission.<init>(FilePermission.java:277)
  316. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  317. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  318. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  319. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  320. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  321. java.nio.file.Files.readAttributes(Files.java:1737)
  322. java.nio.file.Files.size(Files.java:2332)
  323. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  324. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:55)
  325. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:36)
  326. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:89)
  327. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:71)
  328. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  329. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:120)
  330. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:738)
  331. org.elasticsearch.index.store.Store.stats(Store.java:377)
  332. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:969)
  333. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  334. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  335. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  336. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  337. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  338. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  339. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  340. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  341. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  342. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  343. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  344. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  345. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  346. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  347. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  348. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  349. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  350. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  351. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  352. java.lang.Thread.run(Thread.java:748)
  353.  
  354. 39.7% (198.5ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#1]'
  355. 3/10 snapshots sharing following 45 elements
  356. java.io.UnixFileSystem.canonicalize0(Native Method)
  357. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  358. java.io.File.getCanonicalPath(File.java:618)
  359. java.io.FilePermission$1.run(FilePermission.java:215)
  360. java.io.FilePermission$1.run(FilePermission.java:203)
  361. java.security.AccessController.doPrivileged(Native Method)
  362. java.io.FilePermission.init(FilePermission.java:203)
  363. java.io.FilePermission.<init>(FilePermission.java:277)
  364. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  365. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  366. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  367. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  368. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  369. java.nio.file.Files.readAttributes(Files.java:1737)
  370. java.nio.file.Files.size(Files.java:2332)
  371. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  372. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:55)
  373. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:36)
  374. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:89)
  375. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:71)
  376. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  377. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:120)
  378. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:738)
  379. org.elasticsearch.index.store.Store.stats(Store.java:377)
  380. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:969)
  381. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  382. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  383. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  384. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  385. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  386. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  387. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  388. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  389. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  390. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  391. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  392. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  393. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  394. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  395. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  396. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  397. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  398. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  399. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  400. java.lang.Thread.run(Thread.java:748)
  401. 7/10 snapshots sharing following 19 elements
  402. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:353)
  403. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:312)
  404. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:303)
  405. org.elasticsearch.node.NodeService.stats(NodeService.java:112)
  406. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  407. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  408. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140)
  409. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:260)
  410. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:256)
  411. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  412. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  413. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  414. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  415. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  416. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  417. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  418. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  419. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  420. java.lang.Thread.run(Thread.java:748)
  421.  
  422. 37.3% (186.6ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#3]'
  423. 5/10 snapshots sharing following 31 elements
  424. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  425. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  426. java.nio.file.Files.readAttributes(Files.java:1737)
  427. java.nio.file.Files.size(Files.java:2332)
  428. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  429. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:55)
  430. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:36)
  431. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:89)
  432. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:71)
  433. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  434. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:120)
  435. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:738)
  436. org.elasticsearch.index.store.Store.stats(Store.java:377)
  437. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:969)
  438. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  439. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:178)
  440. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:48)
  441. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:430)
  442. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:409)
  443. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:396)
  444. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  445. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  446. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  447. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  448. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  449. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  450. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  451. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  452. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  453. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  454. java.lang.Thread.run(Thread.java:748)
  455. 4/10 snapshots sharing following 21 elements
  456. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:402)
  457. org.elasticsearch.index.translog.Translog.stats(Translog.java:833)
  458. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:483)
  459. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:999)
  460. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  461. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:178)
  462. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:48)
  463. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:430)
  464. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:409)
  465. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:396)
  466. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  467. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  468. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  469. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  470. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  471. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  472. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  473. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  474. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  475. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  476. java.lang.Thread.run(Thread.java:748)
  477. unique snapshot
  478. java.util.Collections$UnmodifiableCollection$1.hasNext(Collections.java:1041)
  479. org.elasticsearch.search.suggest.completion.CompletionFieldStats.completionStats(CompletionFieldStats.java:53)
  480. org.elasticsearch.index.shard.IndexShard.completionStats(IndexShard.java:1005)
  481. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:210)
  482. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:178)
  483. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:48)
  484. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:430)
  485. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:409)
  486. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:396)
  487. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  488. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  489. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  490. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  491. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  492. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  493. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  494. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  495. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  496. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  497. java.lang.Thread.run(Thread.java:748)
  498.  
  499. 36.2% (181.1ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#4]'
  500. 10/10 snapshots sharing following 16 elements
  501. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:178)
  502. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:48)
  503. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:430)
  504. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:409)
  505. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:396)
  506. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  507. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  508. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  509. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  510. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  511. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:665)
  512. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  513. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  514. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  515. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  516. java.lang.Thread.run(Thread.java:748)
  517.  
  518. 8.3% (41.4ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][refresh][T#1]'
  519. 2/10 snapshots sharing following 28 elements
  520. org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:470)
  521. org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
  522. org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
  523. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:491)
  524. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  525. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  526. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  527. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  528. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  529. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  530. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  531. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  532. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  533. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  534. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  535. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  536. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  537. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  538. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  539. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  540. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  541. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  542. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  543. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  544. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  545. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  546. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  547. java.lang.Thread.run(Thread.java:748)
  548.  
  549. 2.6% (13ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][refresh][T#2]'
  550. 2/10 snapshots sharing following 38 elements
  551. sun.nio.ch.FileDispatcherImpl.size0(Native Method)
  552. sun.nio.ch.FileDispatcherImpl.size(FileDispatcherImpl.java:88)
  553. sun.nio.ch.FileChannelImpl.size(FileChannelImpl.java:310)
  554. org.apache.lucene.store.NativeFSLockFactory$NativeFSLock.ensureValid(NativeFSLockFactory.java:182)
  555. org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:43)
  556. org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
  557. org.apache.lucene.codecs.lucene70.Lucene70NormsConsumer.<init>(Lucene70NormsConsumer.java:45)
  558. org.apache.lucene.codecs.lucene70.Lucene70NormsFormat.normsConsumer(Lucene70NormsFormat.java:85)
  559. org.apache.lucene.index.DefaultIndexingChain.writeNorms(DefaultIndexingChain.java:290)
  560. org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:127)
  561. org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:470)
  562. org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
  563. org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
  564. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:491)
  565. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  566. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  567. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  568. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  569. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  570. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  571. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  572. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  573. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  574. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  575. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  576. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  577. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  578. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  579. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  580. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  581. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  582. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  583. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  584. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  585. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  586. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  587. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  588. java.lang.Thread.run(Thread.java:748)
  589. 2/10 snapshots sharing following 37 elements
  590. org.apache.lucene.util.packed.DirectMonotonicWriter.<init>(DirectMonotonicWriter.java:56)
  591. org.apache.lucene.util.packed.DirectMonotonicWriter.getInstance(DirectMonotonicWriter.java:136)
  592. org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addTermsDict(Lucene70DocValuesConsumer.java:448)
  593. org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.doAddSortedField(Lucene70DocValuesConsumer.java:437)
  594. org.apache.lucene.codecs.lucene70.Lucene70DocValuesConsumer.addSortedSetField(Lucene70DocValuesConsumer.java:577)
  595. org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addSortedSetField(PerFieldDocValuesFormat.java:129)
  596. org.apache.lucene.index.SortedSetDocValuesWriter.flush(SortedSetDocValuesWriter.java:221)
  597. org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:249)
  598. org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:133)
  599. org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:470)
  600. org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
  601. org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
  602. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:491)
  603. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  604. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  605. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  606. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  607. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  608. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  609. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  610. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  611. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  612. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  613. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  614. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  615. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  616. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  617. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  618. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  619. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  620. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  621. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  622. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  623. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  624. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  625. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  626. java.lang.Thread.run(Thread.java:748)
  627. 2/10 snapshots sharing following 41 elements
  628. sun.nio.fs.UnixNativeDispatcher.stat0(Native Method)
  629. sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286)
  630. sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70)
  631. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52)
  632. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  633. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  634. java.nio.file.Files.readAttributes(Files.java:1737)
  635. java.nio.file.Files.size(Files.java:2332)
  636. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  637. org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:68)
  638. org.apache.lucene.store.FilterDirectory.fileLength(FilterDirectory.java:68)
  639. org.apache.lucene.index.SegmentCommitInfo.sizeInBytes(SegmentCommitInfo.java:217)
  640. org.apache.lucene.index.DocumentsWriterPerThread.sealFlushedSegment(DocumentsWriterPerThread.java:568)
  641. org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:515)
  642. org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
  643. org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
  644. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:491)
  645. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  646. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  647. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  648. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  649. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  650. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  651. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  652. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  653. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  654. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  655. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  656. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  657. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  658. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  659. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  660. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  661. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  662. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  663. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  664. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  665. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  666. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  667. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  668. java.lang.Thread.run(Thread.java:748)
  669. 2/10 snapshots sharing following 46 elements
  670. sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method)
  671. sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:141)
  672. sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:231)
  673. sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
  674. java.nio.file.Files.delete(Files.java:1126)
  675. org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371)
  676. org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340)
  677. org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
  678. org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:175)
  679. org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63)
  680. org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:748)
  681. org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:753)
  682. org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38)
  683. org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696)
  684. org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690)
  685. org.apache.lucene.index.IndexFileDeleter.deleteNewFiles(IndexFileDeleter.java:664)
  686. org.apache.lucene.index.IndexWriter.deleteNewFiles(IndexWriter.java:4983)
  687. org.apache.lucene.index.IndexWriter.access$200(IndexWriter.java:211)
  688. org.apache.lucene.index.IndexWriter$1.lambda$deleteUnusedFiles$0(IndexWriter.java:355)
  689. org.apache.lucene.index.IndexWriter$1$$Lambda$3016/1130868801.process(Unknown Source)
  690. org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5065)
  691. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:504)
  692. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  693. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  694. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  695. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  696. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  697. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  698. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  699. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  700. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  701. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  702. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  703. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  704. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  705. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  706. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  707. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  708. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  709. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  710. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  711. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  712. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  713. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  714. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  715. java.lang.Thread.run(Thread.java:748)
  716. 2/10 snapshots sharing following 9 elements
  717. sun.misc.Unsafe.park(Native Method)
  718. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  719. java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:734)
  720. java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
  721. java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1277)
  722. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
  723. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
  724. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  725. java.lang.Thread.run(Thread.java:748)
  726.  
  727. 0.0% (94micros out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][[timer]]'
  728. 10/10 snapshots sharing following 2 elements
  729. java.lang.Thread.sleep(Native Method)
  730. org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:543)
  731.  
  732. 0.0% (82.2micros out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][refresh][T#4]'
  733. 2/10 snapshots sharing following 28 elements
  734. org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:470)
  735. org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:554)
  736. org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:719)
  737. org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:491)
  738. org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:288)
  739. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:263)
  740. org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:253)
  741. org.apache.lucene.index.FilterDirectoryReader.doOpenIfChanged(FilterDirectoryReader.java:104)
  742. org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
  743. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:156)
  744. org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
  745. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  746. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  747. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:284)
  748. org.elasticsearch.index.engine.InternalEngine$ExternalSearcherManager.refreshIfNeeded(InternalEngine.java:259)
  749. org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
  750. org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:253)
  751. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1396)
  752. org.elasticsearch.index.engine.InternalEngine.refresh(InternalEngine.java:1375)
  753. org.elasticsearch.index.shard.IndexShard.refresh(IndexShard.java:880)
  754. org.elasticsearch.index.IndexService.maybeRefreshEngine(IndexService.java:696)
  755. org.elasticsearch.index.IndexService.access$400(IndexService.java:97)
  756. org.elasticsearch.index.IndexService$AsyncRefreshTask.runInternal(IndexService.java:898)
  757. org.elasticsearch.index.IndexService$BaseAsyncTask.run(IndexService.java:808)
  758. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  759. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  760. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  761. java.lang.Thread.run(Thread.java:748)
  762.  
  763. 0.0% (19micros out of 500ms) cpu usage by thread 'ticker-schedule-trigger-engine'
  764. 10/10 snapshots sharing following 2 elements
  765. java.lang.Thread.sleep(Native Method)
  766. org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:161)
  767.  
  768. 0.0% (0s out of 500ms) cpu usage by thread 'threadDeathWatcher-2-1'
  769. 10/10 snapshots sharing following 4 elements
  770. java.lang.Thread.sleep(Native Method)
  771. io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:152)
  772. io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
  773. java.lang.Thread.run(Thread.java:748)
  774.  
  775. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][generic][T#14]'
  776. 4/10 snapshots sharing following 2 elements
  777. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  778. java.lang.Thread.run(Thread.java:748)
  779.  
  780. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][generic][T#25]'
  781. 10/10 snapshots sharing following 29 elements
  782. java.lang.Object.wait(Native Method)
  783. java.lang.Object.wait(Object.java:502)
  784. org.elasticsearch.index.seqno.ReplicationTracker.waitForLocalCheckpointToAdvance(ReplicationTracker.java:883)
  785. org.elasticsearch.index.seqno.ReplicationTracker.markAllocationIdAsInSync(ReplicationTracker.java:634)
  786. org.elasticsearch.index.shard.IndexShard.markAllocationIdAsInSync(IndexShard.java:1802)
  787. org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$finalizeRecovery$12(RecoverySourceHandler.java:517)
  788. org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$3567/1842353058.run(Unknown Source)
  789. org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$runUnderPrimaryPermit$5(RecoverySourceHandler.java:256)
  790. org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$3544/1739118319.run(Unknown Source)
  791. org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105)
  792. org.elasticsearch.common.util.CancellableThreads.execute(CancellableThreads.java:86)
  793. org.elasticsearch.indices.recovery.RecoverySourceHandler.runUnderPrimaryPermit(RecoverySourceHandler.java:234)
  794. org.elasticsearch.indices.recovery.RecoverySourceHandler.finalizeRecovery(RecoverySourceHandler.java:517)
  795. org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:220)
  796. org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:98)
  797. org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:50)
  798. org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:107)
  799. org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:104)
  800. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  801. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  802. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  803. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  804. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  805. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1605)
  806. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  807. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  808. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  809. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  810. java.lang.Thread.run(Thread.java:748)
  811.  
  812. 0.0% (0s out of 500ms) cpu usage by thread 'process reaper'
  813. 10/10 snapshots sharing following 6 elements
  814. java.lang.UNIXProcess.waitForProcessExit(Native Method)
  815. java.lang.UNIXProcess.lambda$initStreams$3(UNIXProcess.java:289)
  816. java.lang.UNIXProcess$$Lambda$125/270056930.run(Unknown Source)
  817. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  818. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  819. java.lang.Thread.run(Thread.java:748)
  820.  
  821. 0.0% (0s out of 500ms) cpu usage by thread 'DestroyJavaVM'
  822. unique snapshot
  823. unique snapshot
  824. unique snapshot
  825. unique snapshot
  826. unique snapshot
  827. unique snapshot
  828. unique snapshot
  829. unique snapshot
  830. unique snapshot
  831. unique snapshot
  832.  
  833. 0.0% (0s out of 500ms) cpu usage by thread 'ml-cpp-log-tail-thread'
  834. 10/10 snapshots sharing following 6 elements
  835. java.io.FileInputStream.readBytes(Native Method)
  836. java.io.FileInputStream.read(FileInputStream.java:233)
  837. org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105)
  838. org.elasticsearch.xpack.ml.job.process.NativeController.lambda$tailLogsInThread$0(NativeController.java:64)
  839. org.elasticsearch.xpack.ml.job.process.NativeController$$Lambda$1707/2009502388.run(Unknown Source)
  840. java.lang.Thread.run(Thread.java:748)
  841.  
  842. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[keepAlive/6.4.3]'
  843. 10/10 snapshots sharing following 8 elements
  844. sun.misc.Unsafe.park(Native Method)
  845. java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  846. java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  847. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
  848. java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
  849. java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
  850. org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:82)
  851. java.lang.Thread.run(Thread.java:748)
  852.  
  853. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][generic][T#17]'
  854. 10/10 snapshots sharing following 20 elements
  855. sun.misc.Unsafe.park(Native Method)
  856. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  857. java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
  858. java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
  859. org.elasticsearch.common.util.concurrent.BaseFuture$Sync.get(BaseFuture.java:234)
  860. org.elasticsearch.common.util.concurrent.BaseFuture.get(BaseFuture.java:69)
  861. org.elasticsearch.common.util.concurrent.FutureUtils.get(FutureUtils.java:70)
  862. org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:54)
  863. org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:49)
  864. org.elasticsearch.action.ActionRequestBuilder.get(ActionRequestBuilder.java:60)
  865. org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryCollector.doCollect(IndexRecoveryCollector.java:76)
  866. org.elasticsearch.xpack.monitoring.collector.Collector.collect(Collector.java:87)
  867. org.elasticsearch.xpack.monitoring.MonitoringService$MonitoringExecution$1.doRun(MonitoringService.java:248)
  868. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  869. java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
  870. java.util.concurrent.FutureTask.run(FutureTask.java:266)
  871. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:624)
  872. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  873. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  874. java.lang.Thread.run(Thread.java:748)
  875.  
  876. 0.0% (0s out of 500ms) cpu usage by thread 'Connection evictor'
  877. 10/10 snapshots sharing following 3 elements
  878. java.lang.Thread.sleep(Native Method)
  879. org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
  880. java.lang.Thread.run(Thread.java:748)
  881.  
  882. 0.0% (0s out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][generic][T#21]'
  883. 10/10 snapshots sharing following 29 elements
  884. java.lang.Object.wait(Native Method)
  885. java.lang.Object.wait(Object.java:502)
  886. org.elasticsearch.index.seqno.ReplicationTracker.waitForLocalCheckpointToAdvance(ReplicationTracker.java:883)
  887. org.elasticsearch.index.seqno.ReplicationTracker.markAllocationIdAsInSync(ReplicationTracker.java:634)
  888. org.elasticsearch.index.shard.IndexShard.markAllocationIdAsInSync(IndexShard.java:1802)
  889. org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$finalizeRecovery$12(RecoverySourceHandler.java:517)
  890. org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$3567/1842353058.run(Unknown Source)
  891. org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$runUnderPrimaryPermit$5(RecoverySourceHandler.java:256)
  892. org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$3544/1739118319.run(Unknown Source)
  893. org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:105)
  894. org.elasticsearch.common.util.CancellableThreads.execute(CancellableThreads.java:86)
  895. org.elasticsearch.indices.recovery.RecoverySourceHandler.runUnderPrimaryPermit(RecoverySourceHandler.java:234)
  896. org.elasticsearch.indices.recovery.RecoverySourceHandler.finalizeRecovery(RecoverySourceHandler.java:517)
  897. org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:220)
  898. org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:98)
  899. org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:50)
  900. org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:107)
  901. org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:104)
  902. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  903. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251)
  904. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  905. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309)
  906. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  907. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1605)
  908. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
  909. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  910. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  911. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  912. java.lang.Thread.run(Thread.java:748)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!