> ::: {es-2}{x3-HtrkZRv2TbmluIl8A5Q}{XbAMW1_IQBCyOf-agGJuDA}{ip}{ip:9300}
>    Hot threads at 2017-05-11T15:20:32.077Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
>    
>    18.0% (90.1ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#4]'
>      2/10 snapshots sharing following 5 elements
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      4/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>    
>    15.1% (75.6ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#1]'
>      2/10 snapshots sharing following 25 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:143)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:113)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:939)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:908)
>        org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:113)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1658)
>        org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897)
>        org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:627)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      4/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>    
>    14.3% (71.6ms out of 500ms) cpu usage by thread 'elasticsearch[es-2][bulk][T#2]'
>      4/10 snapshots sharing following 16 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      5/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)

> ::: {elk}{cUzRnyEvRae4M8pyO67PgA}{sb0uq08STgunIYU2HGFRGQ}{ip}{ip:9300}
>    Hot threads at 2017-05-11T15:20:33.501Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
>    
>     0.0% (70.4micros out of 500ms) cpu usage by thread 'elasticsearch[elk][[timer]]'
>      10/10 snapshots sharing following 2 elements
>        java.lang.Thread.sleep(Native Method)
>        org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.run(ThreadPool.java:536)

> ::: {es-3}{1wYo1ev5RfqdVtEWMi3kCA}{syeNlKqMRUevfcq4NeboMA}{ip}{ip:9300}
>    Hot threads at 2017-05-11T15:20:32.075Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
>    
>    18.3% (91.5ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#1]'
>      3/10 snapshots sharing following 16 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      2/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>    
>    17.0% (84.8ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#3]'
>      2/10 snapshots sharing following 30 elements
>        sun.nio.ch.FileDispatcherImpl.force0(Native Method)
>        sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:76)
>        sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388)
>        org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:272)
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
>        org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
>        org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
>        org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
>        org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      4/10 snapshots sharing following 16 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      3/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      unique snapshot
>        sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
>        sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
>        sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
>        io.netty.channel.nio.NioEventLoop.wakeup(NioEventLoop.java:687)
>        io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:753)
>        io.netty.channel.AbstractChannelHandlerContext.safeExecute(AbstractChannelHandlerContext.java:1007)
>        io.netty.channel.AbstractChannelHandlerContext.write(AbstractChannelHandlerContext.java:825)
>        io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:794)
>        io.netty.channel.AbstractChannelHandlerContext.writeAndFlush(AbstractChannelHandlerContext.java:831)
>        io.netty.channel.DefaultChannelPipeline.writeAndFlush(DefaultChannelPipeline.java:1032)
>        io.netty.channel.AbstractChannel.writeAndFlush(AbstractChannel.java:296)
>        org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:411)
>        org.elasticsearch.transport.netty4.Netty4Transport.sendMessage(Netty4Transport.java:94)
>        org.elasticsearch.transport.TcpTransport.internalSendMessage(TcpTransport.java:1089)
>        org.elasticsearch.transport.TcpTransport.sendResponse(TcpTransport.java:1164)
>        org.elasticsearch.transport.TcpTransport.sendResponse(TcpTransport.java:1132)
>        org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:67)
>        org.elasticsearch.transport.TcpTransportChannel.sendResponse(TcpTransportChannel.java:61)
>        org.elasticsearch.transport.DelegatingTransportChannel.sendResponse(DelegatingTransportChannel.java:60)
>        org.elasticsearch.transport.RequestHandlerRegistry$TransportChannelWrapper.sendResponse(RequestHandlerRegistry.java:111)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$ResponseListener.onResponse(TransportReplicationAction.java:581)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction$ResponseListener.onResponse(TransportReplicationAction.java:572)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaResult.respond(TransportReplicationAction.java:434)
>        org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.respondIfPossible(TransportWriteAction.java:179)
>        org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.respond(TransportWriteAction.java:170)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:496)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>    
>    14.1% (70.4ms out of 500ms) cpu usage by thread 'elasticsearch[es-3][bulk][T#4]'
>      2/10 snapshots sharing following 30 elements
>        sun.nio.ch.FileDispatcherImpl.force0(Native Method)
>        sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:76)
>        sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388)
>        org.elasticsearch.index.translog.TranslogWriter.syncUpTo(TranslogWriter.java:272)
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
>        org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
>        org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
>        org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
>        org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      6/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      2/10 snapshots sharing following 5 elements
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)

> ::: {es-1}{-NtWQxXcT8SyJ7bT6tcMfQ}{yQIPqr1ZTYCoaACRyhRzvQ}{ip}{ip:9300}
>    Hot threads at 2017-05-11T15:20:32.093Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
>    
>    19.8% (99.2ms out of 500ms) cpu usage by thread 'elasticsearch[es-1][bulk][T#2]'
>      2/10 snapshots sharing following 26 elements
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:520)
>        org.elasticsearch.index.translog.Translog.ensureSynced(Translog.java:545)
>        org.elasticsearch.index.shard.IndexShard$1.write(IndexShard.java:1686)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.processList(AsyncIOProcessor.java:107)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.drainAndProcess(AsyncIOProcessor.java:99)
>        org.elasticsearch.common.util.concurrent.AsyncIOProcessor.put(AsyncIOProcessor.java:82)
>        org.elasticsearch.index.shard.IndexShard.sync(IndexShard.java:1708)
>        org.elasticsearch.action.support.replication.TransportWriteAction$AsyncAfterWriteAction.run(TransportWriteAction.java:307)
>        org.elasticsearch.action.support.replication.TransportWriteAction$WriteReplicaResult.<init>(TransportWriteAction.java:163)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:406)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      3/10 snapshots sharing following 10 elements
>        sun.misc.Unsafe.park(Native Method)
>        java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:737)
>        java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:647)
>        java.util.concurrent.LinkedTransferQueue.take(LinkedTransferQueue.java:1269)
>        org.elasticsearch.common.util.concurrent.SizeBlockingQueue.take(SizeBlockingQueue.java:161)
>        java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      2/10 snapshots sharing following 17 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:376)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnReplica(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:494)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.onResponse(TransportReplicationAction.java:467)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1674)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1519)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)
>      2/10 snapshots sharing following 25 elements
>        org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:143)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:113)
>        org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:69)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:939)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:908)
>        org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:113)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885)
>        org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:147)
>        org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1658)
>        org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897)
>        org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260)
>        org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252)
>        org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69)
>        org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:627)
>        org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638)
>        org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
>        java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>        java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        java.lang.Thread.run(Thread.java:745)