API tools faq
paste
Guest User

Untitled

a guest
Apr 30th, 2019
226
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 68.41 KB | None | 0 0
raw download clone embed print report
  1. ::: {elastic-data-2}{p5ZZL7h0SsO8_2HmyXEw4w}{-_h5vuRwRU6JfftVGBPkRg}{10.10.10.38}{10.10.10.38:9300}{ml.machine_memory=67554168832, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
  2. Hot threads at 2019-04-30T06:33:08.429Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  3.  
  4. 90.3% (451.7ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-2][management][T#5]'
  5. 3/10 snapshots sharing following 31 elements
  6. java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
  7. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  8. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  9. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  10. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  11. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  12. java.nio.file.Files.readAttributes(Files.java:1737)
  13. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  14. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  15. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  16. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  17. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  18. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  19. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  20. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  21. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  22. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  23. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  24. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  25. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  26. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  27. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  28. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  29. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  30. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  31. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  32. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  33. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  34. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  35. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  36. java.lang.Thread.run(Thread.java:748)
  37. 3/10 snapshots sharing following 20 elements
  38. org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:849)
  39. org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:831)
  40. org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1051)
  41. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:213)
  42. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  43. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  44. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  45. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  46. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  47. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  48. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  49. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  50. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  51. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  52. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  53. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  54. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  55. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  56. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  57. java.lang.Thread.run(Thread.java:748)
  58. 3/10 snapshots sharing following 38 elements
  59. java.io.UnixFileSystem.canonicalize0(Native Method)
  60. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  61. java.io.File.getCanonicalPath(File.java:618)
  62. java.io.FilePermission$1.run(FilePermission.java:215)
  63. java.io.FilePermission$1.run(FilePermission.java:203)
  64. java.security.AccessController.doPrivileged(Native Method)
  65. java.io.FilePermission.init(FilePermission.java:203)
  66. java.io.FilePermission.<init>(FilePermission.java:277)
  67. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  68. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  69. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  70. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  71. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  72. java.nio.file.Files.readAttributes(Files.java:1737)
  73. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  74. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  75. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  76. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  77. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  78. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  79. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  80. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  81. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  82. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  83. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  84. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  85. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  86. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  87. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  88. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  89. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  90. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  91. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  92. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  93. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  94. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  95. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  96. java.lang.Thread.run(Thread.java:748)
  97. unique snapshot
  98. sun.nio.fs.UnixNativeDispatcher.stat0(Native Method)
  99. sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286)
  100. sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70)
  101. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52)
  102. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  103. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  104. java.nio.file.Files.readAttributes(Files.java:1737)
  105. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  106. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  107. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  108. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  109. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  110. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  111. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  112. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  113. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  114. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  115. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  116. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  117. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  118. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  119. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  120. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  121. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  122. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  123. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  124. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  125. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  126. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  127. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  128. java.lang.Thread.run(Thread.java:748)
  129.  
  130. 90.3% (451.4ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-2][management][T#4]'
  131. 3/10 snapshots sharing following 31 elements
  132. java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
  133. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  134. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  135. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  136. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  137. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  138. java.nio.file.Files.readAttributes(Files.java:1737)
  139. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  140. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  141. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  142. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  143. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  144. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  145. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  146. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  147. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  148. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  149. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  150. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  151. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  152. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  153. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  154. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  155. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  156. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  157. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  158. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  159. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  160. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  161. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  162. java.lang.Thread.run(Thread.java:748)
  163. 3/10 snapshots sharing following 20 elements
  164. org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:849)
  165. org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:831)
  166. org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1051)
  167. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:213)
  168. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  169. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  170. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  171. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  172. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  173. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  174. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  175. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  176. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  177. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  178. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  179. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  180. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  181. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  182. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  183. java.lang.Thread.run(Thread.java:748)
  184. 2/10 snapshots sharing following 33 elements
  185. java.security.AccessController.doPrivileged(Native Method)
  186. java.io.FilePermission.init(FilePermission.java:203)
  187. java.io.FilePermission.<init>(FilePermission.java:277)
  188. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  189. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  190. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  191. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  192. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  193. java.nio.file.Files.readAttributes(Files.java:1737)
  194. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  195. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  196. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  197. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  198. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  199. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  200. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  201. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  202. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  203. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  204. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  205. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  206. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  207. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  208. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  209. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  210. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  211. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  212. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  213. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  214. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  215. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  216. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  217. java.lang.Thread.run(Thread.java:748)
  218. 2/10 snapshots sharing following 16 elements
  219. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  220. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  221. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  222. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  223. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  224. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  225. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  226. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  227. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  228. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  229. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  230. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  231. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  232. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  233. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  234. java.lang.Thread.run(Thread.java:748)
  235.  
  236. ::: {elastic-data-1}{3BrlTlQySAGpSG2EAzORyg}{Voq40ON4QsKPFb9Zu2O-ug}{10.10.10.39}{10.10.10.39:9300}{ml.machine_memory=67554168832, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}
  237. Hot threads at 2019-04-30T06:33:08.429Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  238.  
  239. 42.1% (210.7ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][management][T#1]'
  240. 6/10 snapshots sharing following 42 elements
  241. java.io.UnixFileSystem.canonicalize0(Native Method)
  242. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  243. java.io.File.getCanonicalPath(File.java:618)
  244. java.io.FilePermission$1.run(FilePermission.java:215)
  245. java.io.FilePermission$1.run(FilePermission.java:203)
  246. java.security.AccessController.doPrivileged(Native Method)
  247. java.io.FilePermission.init(FilePermission.java:203)
  248. java.io.FilePermission.<init>(FilePermission.java:277)
  249. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  250. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  251. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  252. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  253. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  254. java.nio.file.Files.readAttributes(Files.java:1737)
  255. java.nio.file.Files.size(Files.java:2332)
  256. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  257. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  258. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  259. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  260. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  261. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  262. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  263. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  264. org.elasticsearch.index.store.Store.stats(Store.java:359)
  265. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  266. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  267. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  268. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  269. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  270. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  271. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  272. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  273. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  274. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  275. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  276. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  277. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  278. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  279. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  280. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  281. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  282. java.lang.Thread.run(Thread.java:748)
  283. 3/10 snapshots sharing following 38 elements
  284. java.io.UnixFileSystem.canonicalize0(Native Method)
  285. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  286. java.io.File.getCanonicalPath(File.java:618)
  287. java.io.FilePermission$1.run(FilePermission.java:215)
  288. java.io.FilePermission$1.run(FilePermission.java:203)
  289. java.security.AccessController.doPrivileged(Native Method)
  290. java.io.FilePermission.init(FilePermission.java:203)
  291. java.io.FilePermission.<init>(FilePermission.java:277)
  292. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  293. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  294. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  295. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  296. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  297. java.nio.file.Files.readAttributes(Files.java:1737)
  298. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  299. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  300. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  301. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  302. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  303. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  304. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  305. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  306. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  307. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  308. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  309. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  310. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  311. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  312. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  313. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  314. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  315. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  316. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  317. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  318. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  319. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  320. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  321. java.lang.Thread.run(Thread.java:748)
  322. unique snapshot
  323. org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.ramBytesUsed(BlockTreeTermsReader.java:294)
  324. org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.ramBytesUsed(PerFieldPostingsFormat.java:326)
  325. org.apache.lucene.index.CodecReader.ramBytesUsed(CodecReader.java:214)
  326. org.elasticsearch.index.engine.Engine.fillSegmentStats(Engine.java:849)
  327. org.elasticsearch.index.engine.Engine.segmentsStats(Engine.java:831)
  328. org.elasticsearch.index.shard.IndexShard.segmentStats(IndexShard.java:1051)
  329. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:213)
  330. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  331. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  332. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  333. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  334. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  335. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  336. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  337. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  338. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  339. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  340. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  341. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  342. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  343. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  344. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  345. java.lang.Thread.run(Thread.java:748)
  346.  
  347. 40.6% (202.9ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][management][T#3]'
  348. 7/10 snapshots sharing following 37 elements
  349. java.security.AccessController.doPrivileged(Native Method)
  350. java.io.FilePermission.init(FilePermission.java:203)
  351. java.io.FilePermission.<init>(FilePermission.java:277)
  352. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  353. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  354. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  355. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  356. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  357. java.nio.file.Files.readAttributes(Files.java:1737)
  358. java.nio.file.Files.size(Files.java:2332)
  359. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  360. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  361. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  362. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  363. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  364. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  365. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  366. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  367. org.elasticsearch.index.store.Store.stats(Store.java:359)
  368. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  369. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  370. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  371. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  372. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  373. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  374. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  375. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  376. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  377. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  378. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  379. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  380. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  381. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  382. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  383. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  384. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  385. java.lang.Thread.run(Thread.java:748)
  386. 2/10 snapshots sharing following 35 elements
  387. sun.nio.fs.UnixNativeDispatcher.stat0(Native Method)
  388. sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286)
  389. sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70)
  390. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52)
  391. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  392. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  393. java.nio.file.Files.readAttributes(Files.java:1737)
  394. java.nio.file.Files.size(Files.java:2332)
  395. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  396. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  397. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  398. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  399. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  400. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  401. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  402. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  403. org.elasticsearch.index.store.Store.stats(Store.java:359)
  404. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  405. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  406. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  407. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  408. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  409. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  410. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  411. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  412. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  413. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  414. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  415. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  416. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  417. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  418. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  419. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  420. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  421. java.lang.Thread.run(Thread.java:748)
  422. unique snapshot
  423. sun.nio.fs.UnixNativeDispatcher.stat0(Native Method)
  424. sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286)
  425. sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70)
  426. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52)
  427. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  428. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  429. java.nio.file.Files.readAttributes(Files.java:1737)
  430. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  431. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  432. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  433. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  434. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  435. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  436. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  437. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  438. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  439. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  440. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  441. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  442. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  443. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  444. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  445. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  446. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  447. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  448. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  449. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  450. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  451. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  452. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  453. java.lang.Thread.run(Thread.java:748)
  454.  
  455. 30.1% (150.6ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-1][management][T#4]'
  456. 9/10 snapshots sharing following 45 elements
  457. java.io.UnixFileSystem.canonicalize0(Native Method)
  458. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  459. java.io.File.getCanonicalPath(File.java:618)
  460. java.io.FilePermission$1.run(FilePermission.java:215)
  461. java.io.FilePermission$1.run(FilePermission.java:203)
  462. java.security.AccessController.doPrivileged(Native Method)
  463. java.io.FilePermission.init(FilePermission.java:203)
  464. java.io.FilePermission.<init>(FilePermission.java:277)
  465. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  466. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  467. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  468. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  469. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  470. java.nio.file.Files.readAttributes(Files.java:1737)
  471. java.nio.file.Files.size(Files.java:2332)
  472. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  473. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  474. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  475. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  476. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  477. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  478. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  479. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  480. org.elasticsearch.index.store.Store.stats(Store.java:359)
  481. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  482. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  483. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
  484. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
  485. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
  486. org.elasticsearch.node.NodeService.stats(NodeService.java:109)
  487. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  488. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  489. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
  490. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
  491. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
  492. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  493. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  494. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  495. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  496. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  497. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  498. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  499. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  500. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  501. java.lang.Thread.run(Thread.java:748)
  502. unique snapshot
  503. sun.nio.fs.UnixNativeDispatcher.stat0(Native Method)
  504. sun.nio.fs.UnixNativeDispatcher.stat(UnixNativeDispatcher.java:286)
  505. sun.nio.fs.UnixFileAttributes.get(UnixFileAttributes.java:70)
  506. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:52)
  507. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  508. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  509. java.nio.file.Files.readAttributes(Files.java:1737)
  510. java.nio.file.Files.size(Files.java:2332)
  511. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  512. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  513. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  514. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  515. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  516. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  517. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  518. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  519. org.elasticsearch.index.store.Store.stats(Store.java:359)
  520. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  521. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  522. org.elasticsearch.indices.IndicesService.indexShardStats(IndicesService.java:403)
  523. org.elasticsearch.indices.IndicesService.statsByShard(IndicesService.java:357)
  524. org.elasticsearch.indices.IndicesService.stats(IndicesService.java:348)
  525. org.elasticsearch.node.NodeService.stats(NodeService.java:109)
  526. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74)
  527. org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39)
  528. org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:138)
  529. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:259)
  530. org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:255)
  531. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  532. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  533. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  534. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  535. org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:686)
  536. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  537. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  538. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  539. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  540. java.lang.Thread.run(Thread.java:748)
  541.  
  542. ::: {elastic-data-0}{0McTAmanQsiWYKNJpVYC-A}{HW4KR8qSTSyd66NLApPmtg}{10.10.10.40}{10.10.10.40:9300}{ml.machine_memory=67554168832, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}
  543. Hot threads at 2019-04-30T06:33:08.430Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
  544.  
  545. 67.9% (339.3ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][[169][3]: Lucene Merge Thread #134]'
  546. 4/10 snapshots sharing following 15 elements
  547. org.apache.lucene.store.RateLimitedIndexOutput.writeByte(RateLimitedIndexOutput.java:65)
  548. org.apache.lucene.codecs.compressing.LZ4.encodeLiterals(LZ4.java:147)
  549. org.apache.lucene.codecs.compressing.LZ4.encodeSequence(LZ4.java:168)
  550. org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  551. org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  552. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:241)
  553. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  554. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:645)
  555. org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:215)
  556. org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:104)
  557. org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
  558. org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
  559. org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  560. org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  561. org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  562. 3/10 snapshots sharing following 13 elements
  563. org.apache.lucene.codecs.compressing.LZ4.decompress(LZ4.java:133)
  564. org.apache.lucene.codecs.compressing.CompressionMode$4.decompress(CompressionMode.java:138)
  565. org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.doReset(CompressingStoredFieldsReader.java:483)
  566. org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader$BlockState.reset(CompressingStoredFieldsReader.java:389)
  567. org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.document(CompressingStoredFieldsReader.java:568)
  568. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:641)
  569. org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:215)
  570. org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:104)
  571. org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
  572. org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
  573. org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  574. org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  575. org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  576. 2/10 snapshots sharing following 12 elements
  577. org.apache.lucene.codecs.compressing.LZ4.compress(LZ4.java:241)
  578. org.apache.lucene.codecs.compressing.CompressionMode$LZ4FastCompressor.compress(CompressionMode.java:164)
  579. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.flush(CompressingStoredFieldsWriter.java:241)
  580. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.finishDocument(CompressingStoredFieldsWriter.java:168)
  581. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:645)
  582. org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:215)
  583. org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:104)
  584. org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
  585. org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
  586. org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  587. org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  588. org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  589. unique snapshot
  590. org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
  591. org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.merge(CompressingStoredFieldsWriter.java:643)
  592. org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:215)
  593. org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:104)
  594. org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4482)
  595. org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4077)
  596. org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:625)
  597. org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:101)
  598. org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:662)
  599.  
  600. 42.6% (212.7ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#2]'
  601. 10/10 snapshots sharing following 16 elements
  602. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  603. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  604. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  605. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  606. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  607. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  608. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  609. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  610. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  611. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  612. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  613. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  614. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  615. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  616. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  617. java.lang.Thread.run(Thread.java:748)
  618.  
  619. 41.4% (207ms out of 500ms) cpu usage by thread 'elasticsearch[elastic-data-0][management][T#5]'
  620. 7/10 snapshots sharing following 42 elements
  621. java.io.UnixFileSystem.canonicalize0(Native Method)
  622. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  623. java.io.File.getCanonicalPath(File.java:618)
  624. java.io.FilePermission$1.run(FilePermission.java:215)
  625. java.io.FilePermission$1.run(FilePermission.java:203)
  626. java.security.AccessController.doPrivileged(Native Method)
  627. java.io.FilePermission.init(FilePermission.java:203)
  628. java.io.FilePermission.<init>(FilePermission.java:277)
  629. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  630. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  631. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  632. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  633. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  634. java.nio.file.Files.readAttributes(Files.java:1737)
  635. java.nio.file.Files.size(Files.java:2332)
  636. org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
  637. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:56)
  638. org.elasticsearch.index.store.ByteSizeCachingDirectory.access$200(ByteSizeCachingDirectory.java:37)
  639. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:90)
  640. org.elasticsearch.index.store.ByteSizeCachingDirectory$1.refresh(ByteSizeCachingDirectory.java:72)
  641. org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54)
  642. org.elasticsearch.index.store.ByteSizeCachingDirectory.estimateSizeInBytes(ByteSizeCachingDirectory.java:121)
  643. org.elasticsearch.index.store.Store$StoreDirectory.estimateSize(Store.java:712)
  644. org.elasticsearch.index.store.Store.stats(Store.java:359)
  645. org.elasticsearch.index.shard.IndexShard.storeStats(IndexShard.java:1035)
  646. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:180)
  647. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  648. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  649. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  650. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  651. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  652. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  653. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  654. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  655. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  656. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  657. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  658. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  659. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  660. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  661. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  662. java.lang.Thread.run(Thread.java:748)
  663. 3/10 snapshots sharing following 38 elements
  664. java.io.UnixFileSystem.canonicalize0(Native Method)
  665. java.io.UnixFileSystem.canonicalize(UnixFileSystem.java:172)
  666. java.io.File.getCanonicalPath(File.java:618)
  667. java.io.FilePermission$1.run(FilePermission.java:215)
  668. java.io.FilePermission$1.run(FilePermission.java:203)
  669. java.security.AccessController.doPrivileged(Native Method)
  670. java.io.FilePermission.init(FilePermission.java:203)
  671. java.io.FilePermission.<init>(FilePermission.java:277)
  672. java.lang.SecurityManager.checkRead(SecurityManager.java:888)
  673. sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
  674. sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
  675. sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
  676. sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
  677. java.nio.file.Files.readAttributes(Files.java:1737)
  678. java.nio.file.Files.getLastModifiedTime(Files.java:2266)
  679. org.elasticsearch.index.translog.BaseTranslogReader.getLastModifiedTime(BaseTranslogReader.java:147)
  680. org.elasticsearch.index.translog.Translog.findEarliestLastModifiedAge(Translog.java:435)
  681. org.elasticsearch.index.translog.Translog.earliestLastModifiedAge(Translog.java:423)
  682. org.elasticsearch.index.translog.Translog.stats(Translog.java:861)
  683. org.elasticsearch.index.engine.InternalEngine.getTranslogStats(InternalEngine.java:545)
  684. org.elasticsearch.index.shard.IndexShard.translogStats(IndexShard.java:1065)
  685. org.elasticsearch.action.admin.indices.stats.CommonStats.<init>(CommonStats.java:216)
  686. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:185)
  687. org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction.shardOperation(TransportIndicesStatsAction.java:49)
  688. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.onShardOperation(TransportBroadcastByNodeAction.java:436)
  689. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:414)
  690. org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:401)
  691. org.elasticsearch.transport.TransportRequestHandler.messageReceived(TransportRequestHandler.java:30)
  692. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:250)
  693. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  694. org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:308)
  695. org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)
  696. org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1087)
  697. org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751)
  698. org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
  699. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  700. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  701. java.lang.Thread.run(Thread.java:748)
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!